Unified Llama 4 Scout/Maverick BI-V150 deployment bundle

.dockerignore

@@ -0,0 +1,7 @@
+.git
+.gitea
+README.md
+run.sh
+docker-compose.yml
+presets
+*.log

.gitea/workflows/docker-build-push.yml

@@ -0,0 +1,132 @@
+name: Docker Build and Push
+
+on:
+  push:
+    tags:
+      - "v*"
+
+jobs:
+  docker:
+    runs-on: amd64-ubuntu-24.04
+
+    steps:
+      - name: Clone repository
+        run: |
+          git clone "${{ gitea.server_url }}/${{ gitea.repository }}.git" .
+          git checkout "${{ gitea.ref_name }}"
+
+      - name: Set image metadata
+        run: |
+          IMAGE_NAME="$(echo "${{ gitea.repository }}" | tr '[:upper:]' '[:lower:]' | tr '_' '-')"
+          IMAGE="${DOCKER_REGISTRY}/${DOCKER_USERNAME}/${IMAGE_NAME}:${{ gitea.ref_name }}"
+
+          echo "IMAGE_NAME=${IMAGE_NAME}" >> "$GITEA_ENV"
+          echo "IMAGE=${IMAGE}" >> "$GITEA_ENV"
+
+      - name: Load and Validate Task Info
+        run: |
+          set -a
+          . .gitea/workflows/task_info.env
+          set +a
+
+          for name in FRAMEWORK GPU_TYPE TASK_TYPE; do
+            eval "value=\${${name}:-}"
+            if [ "$name" = "FRAMEWORK" ] && [ -z "$value" ]; then
+              echo "${name} is empty in .gitea/workflows/task_info.env"
+              exit 1
+            fi
+
+            echo "${name}=${value}" >> "$GITEA_ENV"
+          done
+
+      - name: Validate Image Verify Metadata
+        run: |
+          if [ -z "${FIXED_TOKEN:-}" ]; then
+            echo "FIXED_TOKEN is not configured on runner"
+            exit 1
+          fi
+
+          if ! response="$(curl --silent --show-error --location --get 'https://modelhub.org.cn/adminApi/image-verify/validate' \
+            --header "Xc-Token: ${FIXED_TOKEN}" \
+            --data-urlencode "gpuType=${GPU_TYPE:-}" \
+            --data-urlencode "taskType=${TASK_TYPE:-}")"; then
+            echo "failed to call image verify validate API"
+            exit 1
+          fi
+
+          VALIDATE_RESPONSE="$response" python3 - <<'PY'
+          import json
+          import os
+          import sys
+
+          raw = os.environ.get("VALIDATE_RESPONSE", "")
+          try:
+              body = json.loads(raw)
+          except json.JSONDecodeError:
+              print("image verify validate API returned invalid JSON")
+              print(raw)
+              sys.exit(1)
+
+          if body.get("code") == 0 and body.get("data") is True:
+              print("image verify metadata validation passed")
+              sys.exit(0)
+
+          message = body.get("message") or "unknown error"
+          print(f"image verify metadata validation failed: {message}")
+          print(raw)
+          sys.exit(1)
+          PY
+
+      - name: Login to Docker Registry
+        run: |
+          echo "$DOCKER_PASSWORD" | docker login "$DOCKER_REGISTRY" \
+            -u "$DOCKER_USERNAME" \
+            --password-stdin
+
+      - name: Build Docker Image
+        run: |
+          docker build -t "$IMAGE" .
+
+      - name: Push Docker Image
+        run: |
+          for attempt in 1 2 3; do
+            echo "Starting docker push attempt ${attempt}/3 for ${IMAGE}"
+            docker push "$IMAGE" &
+            PUSH_PID=$!
+
+            while kill -0 "$PUSH_PID" 2>/dev/null; do
+              echo "docker push is still running at $(date -u '+%Y-%m-%dT%H:%M:%SZ')"
+              sleep 60
+            done
+
+            if wait "$PUSH_PID"; then
+              echo "docker push completed successfully"
+              exit 0
+            fi
+
+            echo "docker push failed on attempt ${attempt}/3"
+            sleep 30
+          done
+
+          echo "docker push failed after 3 attempts"
+          exit 1
+
+      - name: Notify Image Verify
+        run: |
+          if [ -z "${FIXED_TOKEN:-}" ]; then
+            echo "FIXED_TOKEN is not configured on runner"
+            exit 1
+          fi
+
+          curl --silent --show-error --fail-with-body --location --request POST 'https://modelhub.org.cn//adminApi/image-verify' \
+            --header "Xc-Token: ${FIXED_TOKEN}" \
+            --header 'Content-Type: application/json' \
+            --data-raw "{
+              \"framework\": \"${FRAMEWORK}\",
+              \"gpuType\": \"${GPU_TYPE}\",
+              \"imageUrl\": \"${IMAGE}\",
+              \"taskType\": \"${TASK_TYPE}\",
+              \"createBy\": \"${{ gitea.actor }}\",
+              \"repoUrl\": \"${{ gitea.server_url }}/${{ gitea.repository }}\",
+              \"tag\": \"${{ github.ref_name }}\"
+            }"

.gitea/workflows/task_info.env

@@ -0,0 +1,3 @@
+FRAMEWORK=vllm
+GPU_TYPE=Iluvatar_bi-150
+TASK_TYPE=text-generation

.gitignore

@@ -0,0 +1,3 @@
+*.log
+__pycache__/
+*.pyc

Dockerfile

@@ -0,0 +1,50 @@
+# Llama 4 Scout / Maverick on Iluvatar BI-V150
+#
+# 一个镜像同时支持：
+#   - Llama-4-Scout-17B-16E-Instruct（BF16，TP8，8 卡）
+#   - Llama-4-Maverick-17B-128E-Instruct-FP8（FP8，TP8+PP2，16 卡）
+#
+# 模型权重不打进镜像，运行时按 run.sh 的 scout / maverick 预设挂载。
+#
+# 构建：
+#   docker build -t llama4-bi150:2026-10-26 .
+#
+# 运行：
+#   ./run.sh scout
+#   ./run.sh maverick
+#
+# 补丁是针对下面这个精确基础镜像生成的；换基础镜像需要重新生成 diff。
+#   registry.iluvatar.com.cn:10443/customer/sz/vllm0.17.0-4.4.0-x86:v7
+#   (vLLM 0.17.0+corex.20260612070235)
+
+FROM registry.iluvatar.com.cn:10443/customer/sz/vllm0.17.0-4.4.0-x86:v7
+
+ARG IMAGE_VERSION=2026-10-26
+
+LABEL org.opencontainers.image.title="llama4-bi150" \
+      org.opencontainers.image.description="Llama 4 Scout / Maverick on Iluvatar BI-V150 with vLLM 0.17.0+corex" \
+      org.opencontainers.image.version="${IMAGE_VERSION}"
+
+COPY patches /opt/llama4/patches
+COPY apply_patch.py /opt/llama4/apply_patch.py
+COPY entrypoint.sh /usr/local/bin/llama4-entrypoint.sh
+
+RUN chmod +x /usr/local/bin/llama4-entrypoint.sh \
+    && python3 /opt/llama4/apply_patch.py
+
+ENV MODEL_PATH=/models/llama4 \
+    SERVED_NAME=llama4 \
+    HOST=0.0.0.0 \
+    PORT=8091 \
+    TP=8 \
+    PP=1 \
+    MAX_MODEL_LEN=4096 \
+    MAX_NUM_SEQS=1 \
+    GPU_MEM_UTIL=0.90 \
+    ENFORCE_EAGER=1 \
+    PYTHONUNBUFFERED=1
+
+WORKDIR /opt/llama4
+EXPOSE 8091 8092
+
+ENTRYPOINT ["/usr/local/bin/llama4-entrypoint.sh"]

README.md

@@ -0,0 +1,222 @@
+# Llama 4 Scout / Maverick on Iluvatar BI-V150
+
+在 16×Iluvatar BI-V150 32GB 上运行 Llama 4 Scout 与 Maverick 的统一 Docker 部署仓库。
+
+- 基础镜像：`registry.iluvatar.com.cn:10443/customer/sz/vllm0.17.0-4.4.0-x86:v7`（vLLM `0.17.0+corex`）
+- 一个镜像同时支持 Scout 和 Maverick；模型权重不打进镜像，运行时只读挂载
+- 两个模型共用 Llama 4 chunked-attention 补丁；Maverick 额外使用 FP8 量化 / FP8 MoE 补丁
+- CI 使用 ModelHub 标准 Gitea Actions：推送 `v*` 标签后构建、推送镜像并提交审核
+
+| 模型 | 精度 | 并行 | GPU 数 | 默认端口 |
+|---|---|---|---|---|
+| `LLM-Research/Llama-4-Scout-17B-16E-Instruct` | BF16 | TP8 | 8 | 8091 |
+| `LLM-Research/Llama-4-Maverick-17B-128E-Instruct-FP8` | FP8 | TP8 + PP2 | 16 | 8092 |
+
+---
+
+## 1. 目录内容
+
+```text
+llama4-bi150-deploy/
+├── Dockerfile
+├── entrypoint.sh
+├── apply_patch.py              # 幂等应用全部 Llama 4 补丁
+├── patches/
+│   ├── 01_fp8_quant.patch      # Maverick FP8 量化
+│   ├── 02_fp8_moe.patch        # Maverick FP8 MoE
+│   ├── 03_llama4_chunked_attn.patch
+│   └── corexx_fp8_moe.py       # 逐 expert 反量化 BF16 的 FP8 MoE 回退
+├── presets/
+│   ├── scout.env
+│   └── maverick.env
+├── run.sh                      # 构建 + 启动 + 等 /health
+├── docker-compose.yml          # 可选，按 profile 启动
+├── .gitea/workflows/
+│   ├── docker-build-push.yml
+│   └── task_info.env
+└── README.md
+```
+
+---
+
+## 2. 构建与启动
+
+### 2.1 脚本（推荐）
+
+```bash
+# Scout：8 卡，8091
+./run.sh scout
+
+# Maverick：16 卡，8092
+./run.sh maverick
+```
+
+`run.sh` 会先 `docker build`，再删除同名容器、启动新容器，并等待 `/health` 最多 10 分钟。
+
+### 2.2 指定 GPU 和模型目录
+
+所有预设值都可以用环境变量覆盖：
+
+```bash
+# Scout 使用 8-15 号卡
+GPUS=8,9,10,11,12,13,14,15 ./run.sh scout
+
+# Maverick 使用 0-15 号卡
+GPUS=0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15 ./run.sh maverick
+
+# 模型放在其他目录
+MODEL_HOST_DIR=/mnt/disk1/models/Llama-4-Scout-17B-16E-Instruct ./run.sh scout
+
+# 额外挂载一个目录
+EXTRA_DOCKER_ARGS="-v /mnt/disk1:/data:ro" ./run.sh scout
+
+# 传递额外 vLLM 参数
+EXTRA_ARGS="--max-model-len 8192" ./run.sh scout
+
+# 已有镜像，不重新构建
+BUILD=0 IMAGE=registry.example.com/llama4-bi150:v1.0.0 ./run.sh scout
+
+# 只打印 docker run 命令，便于检查
+DRY_RUN=1 ./run.sh scout
+```
+
+### 2.3 Docker Compose（可选）
+
+```bash
+docker compose --profile scout up -d --build
+docker compose --profile maverick up -d --build
+
+# 覆盖模型目录 / GPU
+SCOUT_MODEL_HOST_DIR=/mnt/disk1/models/scout SCOUT_GPUS=8,9,10,11,12,13,14,15 \
+  docker compose --profile scout up -d --build
+```
+
+---
+
+## 3. 可配置项
+
+| 变量 | Scout 默认 | Maverick 默认 | 说明 |
+|---|---|---|---|
+| `IMAGE` | `llama4-bi150:2026-10-26` | 同左 | 镜像名 |
+| `NAME` | `llama4-scout` | `llama4-maverick` | 容器名 |
+| `PORT` | `8091` | `8092` | 宿主机端口 |
+| `MODEL_HOST_DIR` | `.../Llama-4-Scout-17B-16E-Instruct` | `.../Llama-4-Maverick-17B-128E-Instruct-FP8` | 宿主机模型目录 |
+| `MODEL_MOUNT` | `/models/llama4` | 同左 | 容器内挂载点 |
+| `SERVED_NAME` | `llama4-scout` | `llama4-maverick` | OpenAI API 模型名 |
+| `TP` | `8` | `8` | 张量并行 |
+| `PP` | `1` | `2` | 流水线并行 |
+| `GPUS` | `0,...,7` | `0,...,15` | 传给 `CUDA_VISIBLE_DEVICES` |
+| `MAX_MODEL_LEN` | `4096` | `4096` | 最大上下文 |
+| `MAX_NUM_SEQS` | `1` | `1` | 最大并发 |
+| `GPU_MEM_UTIL` | `0.90` | `0.90` | 显存利用率 |
+| `ENFORCE_EAGER` | `1` | `1` | 是否 `--enforce-eager` |
+| `EXTRA_ARGS` | 空 | 空 | 额外 vLLM 参数 |
+| `EXTRA_DOCKER_ARGS` | 空 | 空 | 额外 docker 参数，如 `-v` |
+| `BUILD` | `1` | `1` | `0` 表示跳过构建 |
+| `DRY_RUN` | `0` | `0` | `1` 只打印命令 |
+
+`GPUS` 也可以直接用 `CUDA_VISIBLE_DEVICES` 设置。`run.sh` 会检查 GPU 数是否少于 `TP*PP`，少于时直接报错。
+
+---
+
+## 4. 验证
+
+```bash
+# 健康检查
+curl -s http://127.0.0.1:8091/health
+curl -s http://127.0.0.1:8092/health
+
+# 模型列表
+curl -s http://127.0.0.1:8091/v1/models | python3 -m json.tool
+curl -s http://127.0.0.1:8092/v1/models | python3 -m json.tool
+
+# 文本
+curl -s http://127.0.0.1:8091/v1/chat/completions \
+  -H 'Content-Type: application/json' \
+  -d '{"model":"llama4-scout","messages":[{"role":"user","content":"你好"}],"max_tokens":64}'
+```
+
+Maverick 图片输入测试：
+
+```python
+from PIL import Image
+import base64, io, requests
+
+img = Image.new("RGB", (336, 336), (220, 20, 20))
+buf = io.BytesIO(); img.save(buf, format="PNG")
+b64 = base64.b64encode(buf.getvalue()).decode()
+
+payload = {
+    "model": "llama4-maverick",
+    "messages": [{"role": "user", "content": [
+        {"type": "text", "text": "这张图片主要是什么颜色？只回答颜色。"},
+        {"type": "image_url", "image_url": {"url": "data:image/png;base64," + b64}},
+    ]}],
+    "max_tokens": 32,
+    "temperature": 0,
+}
+print(requests.post("http://127.0.0.1:8092/v1/chat/completions",
+                    json=payload, timeout=300).json()["choices"][0]["message"]["content"])
+```
+
+首次使用统一镜像时，建议把 Scout 和 Maverick 都跑一遍，确认 FP8 补丁没有影响 Scout 的 BF16 路径。
+
+---
+
+## 5. CI / ModelHub 发布
+
+仓库已包含 ModelHub 标准 CI：
+
+```text
+.gitea/workflows/docker-build-push.yml
+.gitea/workflows/task_info.env
+```
+
+触发方式：
+
+```bash
+git tag v1.0.0
+git push origin v1.0.0
+```
+
+CI 会：
+
+1. 在 `amd64-ubuntu-24.04` runner 上 clone 仓库；
+2. 校验 `task_info.env`；
+3. `docker build`；
+4. 推送 `${DOCKER_REGISTRY}/${DOCKER_USERNAME}/llama-4-bi150:v1.0.0`；
+5. 调 ModelHub `image-verify` 接口提交审核。
+
+`task_info.env`：
+
+```bash
+FRAMEWORK=vllm
+GPU_TYPE=Iluvatar_bi-150
+TASK_TYPE=text-generation
+```
+
+Runner 需要配置以下环境变量（不是仓库 secret）：
+
+```text
+DOCKER_REGISTRY
+DOCKER_USERNAME
+DOCKER_PASSWORD
+FIXED_TOKEN
+```
+
+注意事项：
+
+- Runner 必须能拉取 `registry.iluvatar.com.cn:10443` 的基础镜像；如果该仓库需要登录，请在 runner 上预登录，或自行给 workflow 增加基础镜像 registry 的 login step。
+- CI 只负责构建和推送，不负责 GPU / 模型挂载；GPU 和挂载通过 `run.sh` 或 `docker compose` 设置。
+- CI 构建机不需要 GPU，但需要足够的磁盘空间。
+- 如果仓库改名，CI 生成的镜像名会跟着变。
+
+---
+
+## 6. 限制与注意事项
+
+- Scout 需要 8 张卡（TP8）；Maverick 需要 16 张卡（TP8+PP2）。16 卡机器上两者不能同时运行。
+- 当前默认 `MAX_MODEL_LEN=4096`、`MAX_NUM_SEQS=1`，适合功能验证；生产并发需要先调整显存/上下文。
+- 补丁是针对当前精确基础镜像生成的。更换基础镜像后不要直接复用，需要重新生成 diff 并在目标环境验证。
+- `--privileged --pid host -v /dev:/dev` 是 Iluvatar 运行时要求；只给 `/dev/iluvatar*` 可能报 `cudaGetDeviceCount() ... Error 1001`。
+- 模型目录只读挂载，容器不会修改宿主机模型文件。

apply_patch.py

@@ -0,0 +1,117 @@
+#!/usr/bin/env python3
+"""Apply the CoreX vLLM patches needed by Llama 4 Scout / Maverick.
+
+The patches in ./patches were generated against the exact base image
+`registry.iluvatar.com.cn:10443/customer/sz/vllm0.17.0-4.4.0-x86:v7`
+(vLLM 0.17.0+corex.20260612070235).
+
+Scout needs only the chunked-attention patch.  Maverick additionally needs the
+FP8 quantization / FP8 MoE patches and the `corexx_fp8_moe` fallback module.
+One image contains the union of both patch sets; the FP8 paths are only used
+when the mounted model is FP8.
+
+The script is idempotent: it checks for a marker in each target file and skips
+patches that are already applied.
+"""
+
+from __future__ import annotations
+
+import ast
+import os
+import pathlib
+import shutil
+import subprocess
+import sys
+
+PATCH_DIR = pathlib.Path(
+    os.environ.get("LLAMA4_PATCH_DIR", "/opt/llama4/patches")
+)
+
+# LLAMA4_PKG_ROOT is only needed for local testing against a copy of the
+# vendor files.  In the image it is derived from the installed vllm package.
+_pkg_env = os.environ.get("LLAMA4_PKG_ROOT")
+if _pkg_env:
+    PKG = pathlib.Path(_pkg_env)
+else:
+    import vllm
+
+    PKG = pathlib.Path(vllm.__file__).resolve().parent.parent
+
+# patch file -> (target file relative to the package root, applied marker)
+TARGETS = {
+    "01_fp8_quant.patch": (
+        "vllm/_custom_ops.py",
+        "COREXX_FP8_QUANT_PATCH",
+    ),
+    "02_fp8_moe.patch": (
+        "vllm/model_executor/layers/fused_moe/fused_moe.py",
+        "COREXX_FP8_MOE_PATCH",
+    ),
+    "03_llama4_chunked_attn.patch": (
+        "vllm/v1/attention/backends/utils.py",
+        "seq_lens_np=seqlens_k_local",
+    ),
+}
+
+
+def apply_one(patch_name: str, target_rel: str, marker: str) -> None:
+    target = PKG / target_rel
+    if not target.exists():
+        sys.exit(f"target file not found: {target}")
+    if marker in target.read_text():
+        print(f"[skip] {target} (already patched)")
+        return
+
+    patch_file = PATCH_DIR / patch_name
+    if not patch_file.exists():
+        sys.exit(f"missing patch file: {patch_file}")
+
+    result = subprocess.run(
+        [
+            "patch",
+            "-p1",
+            "--forward",
+            "--batch",
+            "--no-backup-if-mismatch",
+            "-d",
+            str(PKG),
+            "-i",
+            str(patch_file),
+        ],
+        text=True,
+        capture_output=True,
+    )
+    if result.stdout:
+        print(result.stdout, end="")
+    if result.returncode != 0:
+        if result.stderr:
+            print(result.stderr, file=sys.stderr)
+        sys.exit(f"patch failed: {patch_name}")
+    if marker not in target.read_text():
+        sys.exit(f"patch {patch_name} applied but marker not found in {target}")
+    print(f"[ok]   {target}")
+
+
+def main() -> None:
+    for patch_name, (target_rel, marker) in TARGETS.items():
+        apply_one(patch_name, target_rel, marker)
+
+    src = PATCH_DIR / "corexx_fp8_moe.py"
+    dst = PKG / "vllm/model_executor/layers/fused_moe/corexx_fp8_moe.py"
+    if not src.exists():
+        sys.exit(f"missing module: {src}")
+    shutil.copy2(src, dst)
+    print(f"[copy] {src} -> {dst}")
+
+    for path in (
+        PKG / "vllm/_custom_ops.py",
+        PKG / "vllm/model_executor/layers/fused_moe/fused_moe.py",
+        dst,
+        PKG / "vllm/v1/attention/backends/utils.py",
+    ):
+        ast.parse(path.read_text())
+    print("all patches applied; syntax OK")
+
+
+if __name__ == "__main__":
+    main()

docker-compose.yml

@@ -0,0 +1,54 @@
+x-llama4-common: &llama4-common
+  build:
+    context: .
+    dockerfile: Dockerfile
+  image: llama4-bi150:${TAG:-2026-10-26}
+  network_mode: host
+  ipc: host
+  privileged: true
+  pid: host
+  ulimits:
+    memlock: -1
+    stack: 67108864
+  restart: unless-stopped
+
+services:
+  scout:
+    <<: *llama4-common
+    profiles: ["scout"]
+    container_name: llama4-scout
+    volumes:
+      - /dev:/dev
+      - ${SCOUT_MODEL_HOST_DIR:-/mnt/disk0/models/LLM-Research/Llama-4-Scout-17B-16E-Instruct}:/models/llama4:ro
+    environment:
+      MODEL_PATH: /models/llama4
+      SERVED_NAME: llama4-scout
+      PORT: "8091"
+      TP: "8"
+      PP: "1"
+      MAX_MODEL_LEN: "4096"
+      MAX_NUM_SEQS: "1"
+      GPU_MEM_UTIL: "0.90"
+      ENFORCE_EAGER: "1"
+      CUDA_VISIBLE_DEVICES: "${SCOUT_GPUS:-0,1,2,3,4,5,6,7}"
+      EXTRA_ARGS: "${SCOUT_EXTRA_ARGS:-}"
+
+  maverick:
+    <<: *llama4-common
+    profiles: ["maverick"]
+    container_name: llama4-maverick
+    volumes:
+      - /dev:/dev
+      - ${MAVERICK_MODEL_HOST_DIR:-/mnt/disk0/models/LLM-Research/Llama-4-Maverick-17B-128E-Instruct-FP8}:/models/llama4:ro
+    environment:
+      MODEL_PATH: /models/llama4
+      SERVED_NAME: llama4-maverick
+      PORT: "8092"
+      TP: "8"
+      PP: "2"
+      MAX_MODEL_LEN: "4096"
+      MAX_NUM_SEQS: "1"
+      GPU_MEM_UTIL: "0.90"
+      ENFORCE_EAGER: "1"
+      CUDA_VISIBLE_DEVICES: "${MAVERICK_GPUS:-0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15}"
+      EXTRA_ARGS: "${MAVERICK_EXTRA_ARGS:-}"

entrypoint.sh

@@ -0,0 +1,71 @@
+#!/usr/bin/env bash
+#
+# Llama 4 Scout / Maverick 统一容器入口：
+#   1. 幂等应用 CoreX vLLM 的 Llama 4 补丁（Scout 只用 chunked attention，
+#      Maverick 额外使用 FP8 补丁；补丁脚本会按 marker 自动跳过已应用项）
+#   2. 检查模型目录
+#   3. exec vLLM（信号可直接传给 vLLM）
+#
+set -euo pipefail
+
+PATCH_SCRIPT="${LLAMA4_PATCH_SCRIPT:-/opt/llama4/apply_patch.py}"
+VLLM_BIN="${VLLM_BIN:-vllm}"
+
+python3 "${PATCH_SCRIPT}"
+
+MODEL_PATH="${MODEL_PATH:-/models/llama4}"
+SERVED_NAME="${SERVED_NAME:-llama4}"
+HOST="${HOST:-0.0.0.0}"
+PORT="${PORT:-8091}"
+TP="${TP:-8}"
+PP="${PP:-1}"
+MAX_MODEL_LEN="${MAX_MODEL_LEN:-4096}"
+MAX_NUM_SEQS="${MAX_NUM_SEQS:-1}"
+GPU_MEM_UTIL="${GPU_MEM_UTIL:-0.90}"
+ENFORCE_EAGER="${ENFORCE_EAGER:-1}"
+EXTRA_ARGS="${EXTRA_ARGS:-}"
+
+if [[ ! -f "${MODEL_PATH}/config.json" ]]; then
+  echo "ERROR: 模型目录不完整：${MODEL_PATH}/config.json 不存在" >&2
+  echo "请确认已挂载模型，例如：" >&2
+  echo "  -v /mnt/disk0/models/LLM-Research/Llama-4-Scout-17B-16E-Instruct:/models/llama4:ro" >&2
+  echo "  -v /mnt/disk0/models/LLM-Research/Llama-4-Maverick-17B-128E-Instruct-FP8:/models/llama4:ro" >&2
+  exit 1
+fi
+
+PP_ARGS=()
+if (( PP > 1 )); then
+  PP_ARGS=(--pipeline-parallel-size "${PP}")
+fi
+
+EAGER_ARGS=()
+if [[ "${ENFORCE_EAGER}" == "1" ]]; then
+  EAGER_ARGS=(--enforce-eager)
+fi
+
+echo "============================================================"
+echo "Llama 4 (BI-V150 unified image)"
+echo "  model      : ${MODEL_PATH}"
+echo "  served name: ${SERVED_NAME}"
+echo "  listen     : ${HOST}:${PORT}"
+echo "  TP / PP    : ${TP} / ${PP}  (total $((TP * PP)) GPUs)"
+echo "  max len    : ${MAX_MODEL_LEN}"
+echo "  max seqs   : ${MAX_NUM_SEQS}"
+echo "  gpu util   : ${GPU_MEM_UTIL}"
+echo "  eager      : ${ENFORCE_EAGER}"
+echo "  visible    : ${CUDA_VISIBLE_DEVICES:-all}"
+echo "  extra      : ${EXTRA_ARGS}"
+echo "============================================================"
+
+# shellcheck disable=SC2086
+exec "${VLLM_BIN}" serve "${MODEL_PATH}" \
+  --host "${HOST}" \
+  --port "${PORT}" \
+  --served-model-name "${SERVED_NAME}" \
+  --tensor-parallel-size "${TP}" \
+  "${PP_ARGS[@]}" \
+  --max-model-len "${MAX_MODEL_LEN}" \
+  --max-num-seqs "${MAX_NUM_SEQS}" \
+  --gpu-memory-utilization "${GPU_MEM_UTIL}" \
+  "${EAGER_ARGS[@]}" \
+  ${EXTRA_ARGS}

patches/01_fp8_quant.patch

@@ -0,0 +1,46 @@
+--- a/vllm/_custom_ops.py
++++ b/vllm/_custom_ops.py
+@@ -1933,14 +1933,28 @@
+     if scale is None:
+         if use_per_token_if_dynamic:
+             scale = torch.empty((shape[0], 1), device=input.device, dtype=torch.float32)
+-            torch.ops._C.dynamic_per_token_scaled_fp8_quant(
+-                output, input, scale, scale_ub
+-            )
++            # COREXX_FP8_QUANT_PATCH: native fallback (vLLM _C op missing on CoreX)
++            m = input.shape[0]
++            s = input.abs().amax(dim=1, keepdim=True).float().clamp_min(1e-12) / 448.0
++            if scale_ub is not None:
++                ub = torch.as_tensor(scale_ub, device=s.device, dtype=s.dtype)
++                s = torch.minimum(s, ub)
++            scale[:m].copy_(s)
++            if shape[0] > m:
++                scale[m:].fill_(1.0)
++                output[m:].zero_()
++            output[:m].copy_((input.float() / s).to(out_dtype))
+         else:
+             scale = torch.empty(1, device=input.device, dtype=torch.float32)
+-            torch.ops._C.dynamic_scaled_fp8_quant(output, input, scale)
++            # COREXX_FP8_QUANT_PATCH: native fallback
++            amax = input.abs().amax().float().clamp_min(1e-12)
++            scale.fill_(amax / 448.0)
++            output.copy_((input.float() / scale).to(out_dtype))
+     else:
+-        torch.ops._C.static_scaled_fp8_quant(output, input, scale, group_shape)
++        # COREXX_FP8_QUANT_PATCH: native fallback
++        if group_shape is not None and tuple(group_shape) not in ((-1, -1), (-1, 1), (1, -1)):
++            raise NotImplementedError(f"native fp8 static quant: unsupported group_shape {group_shape}")
++        output.copy_((input.float() / scale).to(out_dtype))
+ 
+     return output, scale
+ 
+@@ -3824,6 +3838,9 @@
+         mul_routed_weight,
+         top_k,
+         config['BLOCK_SIZE_M'],
++        # COREXX_FP8_MOE_PATCH: pass quant scales to ixformer kernel
++        w_scale=B_scale,
++        a_scale=A_scale,
+         bias=bias
+     )
+ 

patches/02_fp8_moe.patch

@@ -0,0 +1,183 @@
+--- a/vllm/model_executor/layers/fused_moe/fused_moe.py
++++ b/vllm/model_executor/layers/fused_moe/fused_moe.py
+@@ -1645,7 +1645,38 @@
+     # check constraints
+     if use_fp8_w8a8 or use_int8_w8a8 or use_int8_w8a16 or use_int4_w4a16 or w1_scale or \
+         w2_scale or w1_zp or w2_zp or a1_scale or a2_scale:
+-        raise ValueError("Quantized MoE is not supported")
++        # COREXX_FP8_MOE_PATCH: generic fused_experts_impl supports quantized MoE
++        res = fused_experts_impl(
++            hidden_states=hidden_states,
++            w1=w1,
++            w2=w2,
++            topk_weights=topk_weights,
++            topk_ids=topk_ids,
++            inplace=False,
++            activation=activation,
++            apply_router_weight_on_input=apply_router_weight_on_input,
++            use_fp8_w8a8=use_fp8_w8a8,
++            use_int8_w8a8=use_int8_w8a8,
++            use_int8_w8a16=use_int8_w8a16,
++            use_int4_w4a16=use_int4_w4a16,
++            ocp_mx_scheme=ocp_mx_scheme,
++            per_channel_quant=per_channel_quant,
++            global_num_experts=global_num_experts,
++            expert_map=expert_map,
++            w1_scale=w1_scale,
++            w2_scale=w2_scale,
++            w1_zp=w1_zp,
++            w2_zp=w2_zp,
++            a1_scale=a1_scale,
++            a2_scale=a2_scale,
++            block_shape=block_shape,
++            w1_bias=w1_bias,
++            w2_bias=w2_bias,
++        )
++        if output is not None:
++            output.copy_(res)
++            return output
++        return res
+ 
+     attn_metadata = get_forward_context().attn_metadata
+     use_ep = expert_map is not None
+@@ -1912,8 +1943,32 @@
+     w1_bias: torch.Tensor | None = None,
+     w2_bias: torch.Tensor | None = None,
+ ) -> torch.Tensor:
++    # COREXX_FP8_MOE_PATCH: dedicated per-expert BF16 fallback for FP8 MoE
++    if hidden_states.dtype in (
++        torch.float8_e4m3fn, torch.float8_e5m2,
++        torch.float8_e4m3fnuz, torch.float8_e5m2fnuz,
++    ):
++        from vllm.model_executor.layers.fused_moe.corexx_fp8_moe import corexx_fp8_moe
++        return corexx_fp8_moe(
++            hidden_states=hidden_states,
++            w1=w1,
++            w2=w2,
++            topk_weights=topk_weights,
++            topk_ids=topk_ids,
++            activation=activation,
++            apply_router_weight_on_input=apply_router_weight_on_input,
++            global_num_experts=global_num_experts,
++            expert_map=expert_map,
++            w1_scale=w1_scale,
++            w2_scale=w2_scale,
++            a1_scale=a1_scale,
++            a2_scale=a2_scale,
++            block_shape=block_shape,
++            w1_bias=w1_bias,
++            w2_bias=w2_bias,
++        )
++
+     # Convert string activation to enum for internal use
+-    activation_enum = MoEActivation.from_str(activation)
+ 
+     # Check constraints.
+     if use_int4_w4a16:
+@@ -1937,7 +1992,12 @@
+     assert hidden_states.is_contiguous(), "Hidden_states must be contiguous"
+     assert w1.stride(-1) == 1, "Stride of last dimension must be 1"
+     assert w2.stride(-1) == 1, "Stride of last dimension must be 1"
+-    assert hidden_states.dtype in [torch.float32, torch.float16, torch.bfloat16]
++    # COREXX_FP8_MOE_PATCH: modular kernel may pass pre-quantized FP8 input
++    assert hidden_states.dtype in [
++        torch.float32, torch.float16, torch.bfloat16,
++        torch.float8_e4m3fn, torch.float8_e5m2,
++        torch.float8_e4m3fnuz, torch.float8_e5m2fnuz,
++    ]
+ 
+     num_tokens = hidden_states.size(0)
+     E, N, _ = w1.size()
+@@ -1949,6 +2009,9 @@
+     # https://github.com/vllm-project/vllm/issues/5938
+     CHUNK_SIZE = envs.VLLM_FUSED_MOE_CHUNK_SIZE
+     M = min(num_tokens, CHUNK_SIZE)
++    # COREXX_FP8_MOE_PATCH: FP8 input still needs BF16 compute buffers/output
++    io_dtype = hidden_states.dtype if hidden_states.dtype in (
++        torch.float32, torch.float16, torch.bfloat16) else torch.bfloat16
+ 
+     config_dtype = _get_config_dtype_str(
+         use_fp8_w8a8=use_fp8_w8a8,
+@@ -1982,7 +2045,7 @@
+     cache13 = torch.empty(
+         M * top_k_num * max(N, K),
+         device=hidden_states.device,
+-        dtype=hidden_states.dtype,
++        dtype=io_dtype,
+     )
+     intermediate_cache1 = cache13[: M * top_k_num * N].view(M, top_k_num, N)
+     intermediate_cache3 = cache13[: M * top_k_num * K].view(M, top_k_num, K)
+@@ -1994,19 +2057,22 @@
+     intermediate_cache2 = torch.empty(
+         (M * top_k_num, activation_out_dim),
+         device=hidden_states.device,
+-        dtype=hidden_states.dtype,
++        dtype=io_dtype,
+     )
+ 
+-    if hidden_states.dtype == torch.bfloat16:
++    if io_dtype == torch.bfloat16:
+         compute_type = tl.bfloat16
+-    elif hidden_states.dtype == torch.float16:
++    elif io_dtype == torch.float16:
+         compute_type = tl.float16
+-    elif hidden_states.dtype == torch.float32:
++    elif io_dtype == torch.float32:
+         compute_type = tl.float32
+     else:
+-        raise ValueError(f"Unsupported compute_type: {hidden_states.dtype}")
++        raise ValueError(f"Unsupported compute_type: {io_dtype}")
+ 
+-    out_hidden_states = hidden_states if inplace else torch.empty_like(hidden_states)
++    if inplace and io_dtype == hidden_states.dtype:
++        out_hidden_states = hidden_states
++    else:
++        out_hidden_states = torch.empty_like(hidden_states, dtype=io_dtype)
+ 
+     if ocp_mx_scheme is not None:
+         # TODO: On platforms for which `current_platform.supports_mx()` is True
+@@ -2064,14 +2130,24 @@
+ 
+         curr_topk_ids = topk_ids[begin_chunk_idx:end_chunk_idx]
+         curr_topk_weights = topk_weights[begin_chunk_idx:end_chunk_idx]
+-        qcurr_hidden_states, a1q_scale = moe_kernel_quantize_input(
+-            A=curr_hidden_states,
+-            A_scale=a1_scale,
+-            quant_dtype=quant_dtype,
+-            per_act_token_quant=per_channel_quant,
+-            block_shape=block_shape,
+-            ocp_mx_scheme=ocp_mx_scheme,
+-        )
++        # COREXX_FP8_MOE_PATCH: use pre-quantized FP8 input as-is
++        if curr_hidden_states.dtype in (
++            torch.float8_e4m3fn, torch.float8_e5m2,
++            torch.float8_e4m3fnuz, torch.float8_e5m2fnuz,
++        ):
++            qcurr_hidden_states = curr_hidden_states
++            a1q_scale = a1_scale
++            if a1q_scale is None:
++                raise ValueError("pre-quantized MoE input requires a1_scale")
++        else:
++            qcurr_hidden_states, a1q_scale = moe_kernel_quantize_input(
++                A=curr_hidden_states,
++                A_scale=a1_scale,
++                quant_dtype=quant_dtype,
++                per_act_token_quant=per_channel_quant,
++                block_shape=block_shape,
++                ocp_mx_scheme=ocp_mx_scheme,
++            )
+ 
+         # # SPARSITY_FACTOR is a heuristic margin ensuring tokens_in_chunk * top_k
+         # # activates only a small fraction of total experts
+@@ -2305,8 +2381,9 @@
+                                self.quant_config.w2_scale,
+                                self.quant_config.w1_zp,
+                                self.quant_config.w2_zp,
+-                               self.quant_config.a1_scale,
+-                               self.quant_config.a2_scale,
++                               # COREXX_FP8_MOE_PATCH: prefer per-call dynamic scales
++                               a1q_scale if a1q_scale is not None else self.quant_config.a1_scale,
++                               a2_scale if a2_scale is not None else self.quant_config.a2_scale,
+                                self.quant_config.block_shape,
+                                self.quant_config.w1_bias,
+                                self.quant_config.w2_bias,

patches/03_llama4_chunked_attn.patch

@@ -0,0 +1,16 @@
+--- a/vllm/v1/attention/backends/utils.py
++++ b/vllm/v1/attention/backends/utils.py
+@@ -346,7 +346,13 @@
+     return CommonAttentionMetadata(
+         query_start_loc_cpu=query_start_loc_cpu,
+         query_start_loc=query_start_loc_cpu.to(device=device, non_blocking=True),
++        key_start_loc=torch.from_numpy(
++            np.concatenate(
++                (np.zeros(1, dtype=np.int32), np.cumsum(seqlens_k_local, dtype=np.int32))
++            )
++        ).to(device=device, non_blocking=True),
+         seq_lens=seq_lens_cpu.to(device=device, non_blocking=True),
++        seq_lens_np=seqlens_k_local,
+         num_reqs=len(seq_lens_cpu),
+         num_actual_tokens=common_attn_metadata.num_actual_tokens,
+         max_query_len=seqlens_q_local.max(),

patches/corexx_fp8_moe.py

@@ -0,0 +1,125 @@
+"""CoreX fallback for FP8 W8A8 MoE on Iluvatar BI-V150.
+
+The CoreX ixformer MoE kernels are W16A16 only and torch._scaled_mm supports
+only tensor-wise scales, so per-token/per-channel FP8 MoE cannot use them.
+This module dequantizes one expert at a time (each expert shard is only a few
+hundred MB on a TP rank), runs BF16 matmuls, and scatters the result.
+"""
+from __future__ import annotations
+
+import torch
+import torch.nn.functional as F
+
+from vllm.logger import init_logger
+
+logger = init_logger(__name__)
+_FP8_DTYPES = (
+    torch.float8_e4m3fn,
+    torch.float8_e5m2,
+    torch.float8_e4m3fnuz,
+    torch.float8_e5m2fnuz,
+)
+_LOGGED = False
+
+
+def _dequant_weight(w: torch.Tensor, scale: torch.Tensor | None) -> torch.Tensor:
+    """w: [rows, cols] FP8, scale: per-output-row (len rows) or scalar."""
+    if scale is None:
+        return w.to(torch.bfloat16)
+    s = scale.reshape(-1)
+    if s.numel() == 1:
+        return (w.float() * s[0]).to(torch.bfloat16)
+    if s.numel() != w.shape[0]:
+        # try to broadcast a [1, rows] style scale
+        s = s.reshape(w.shape[0], -1)[:, 0]
+    return (w.float() * s.reshape(-1, 1)).to(torch.bfloat16)
+
+
+def corexx_fp8_moe(
+    hidden_states: torch.Tensor,
+    w1: torch.Tensor,
+    w2: torch.Tensor,
+    topk_weights: torch.Tensor,
+    topk_ids: torch.Tensor,
+    activation,
+    apply_router_weight_on_input: bool,
+    global_num_experts: int,
+    expert_map: torch.Tensor | None,
+    w1_scale: torch.Tensor | None,
+    w2_scale: torch.Tensor | None,
+    a1_scale: torch.Tensor | None,
+    a2_scale: torch.Tensor | None,
+    block_shape,
+    w1_bias: torch.Tensor | None,
+    w2_bias: torch.Tensor | None,
+) -> torch.Tensor:
+    global _LOGGED
+    if expert_map is not None:
+        raise NotImplementedError("CoreX FP8 MoE fallback does not support expert_map/EP")
+    if block_shape is not None:
+        raise NotImplementedError("CoreX FP8 MoE fallback does not support block FP8")
+    if not _LOGGED:
+        logger.info(
+            "CoreX FP8 MoE fallback: hidden=%s %s a1_scale=%s w1=%s w1_scale=%s w2=%s w2_scale=%s topk=%s",
+            tuple(hidden_states.shape), hidden_states.dtype,
+            None if a1_scale is None else tuple(a1_scale.shape),
+            tuple(w1.shape), None if w1_scale is None else tuple(w1_scale.shape),
+            tuple(w2.shape), None if w2_scale is None else tuple(w2_scale.shape),
+            tuple(topk_ids.shape),
+        )
+        _LOGGED = True
+
+    if hidden_states.dtype in _FP8_DTYPES:
+        if a1_scale is None:
+            raise ValueError("FP8 MoE input requires a1_scale")
+        A = (hidden_states.float() * a1_scale.reshape(-1, 1)).to(torch.bfloat16)
+    else:
+        A = hidden_states
+
+    E = w1.shape[0]
+    N = w1.shape[1]
+    K = w1.shape[2]
+    I = w2.shape[2]
+    assert N == 2 * I, f"expected w1 N=2*I, got {N} vs {I}"
+
+    act_name = getattr(activation, "value", activation)
+
+    M = A.shape[0]
+    out = torch.zeros((M, K), dtype=torch.bfloat16, device=A.device)
+    topk = topk_ids.shape[1]
+
+    # COREXX_FP8_MOE_PATCH: only touch experts that actually received tokens
+    active_experts = torch.unique(topk_ids).tolist()
+    for e in active_experts:
+        if e < 0 or e >= E:
+            continue
+        # rows of A routed to expert e (global id == local id when expert_map is None)
+        for k in range(topk):
+            idx = (topk_ids[:, k] == e).nonzero(as_tuple=False).flatten()
+            if idx.numel() == 0:
+                continue
+            x = A.index_select(0, idx)
+            if apply_router_weight_on_input:
+                x = x * topk_weights[idx, k].to(x.dtype).unsqueeze(1)
+            w1_e = _dequant_weight(w1[e], None if w1_scale is None else w1_scale[e])
+            h = x @ w1_e.t()
+            if w1_bias is not None:
+                h = h + w1_bias[e]
+            gate = h[:, :I]
+            up = h[:, I:N]
+            if act_name == "silu":
+                act = F.silu(gate) * up
+            elif act_name == "gelu":
+                act = F.gelu(gate) * up
+            else:
+                raise NotImplementedError(f"CoreX FP8 MoE activation {act_name}")
+            w2_e = _dequant_weight(w2[e], None if w2_scale is None else w2_scale[e])
+            y = act @ w2_e.t()
+            if w2_bias is not None:
+                y = y + w2_bias[e]
+            if not apply_router_weight_on_input:
+                y = y * topk_weights[idx, k].to(y.dtype).unsqueeze(1)
+            out.index_add_(0, idx, y)
+            del w1_e, w2_e, h, act, y
+
+    return out

presets/maverick.env

@@ -0,0 +1,12 @@
+# Llama 4 Maverick FP8 默认配置
+DEFAULT_IMAGE="llama4-bi150:2026-10-26"
+DEFAULT_NAME="llama4-maverick"
+DEFAULT_PORT="8092"
+DEFAULT_MODEL_HOST_DIR="/mnt/disk0/models/LLM-Research/Llama-4-Maverick-17B-128E-Instruct-FP8"
+DEFAULT_SERVED_NAME="llama4-maverick"
+DEFAULT_TP="8"
+DEFAULT_PP="2"
+DEFAULT_GPUS="0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15"
+DEFAULT_MAX_MODEL_LEN="4096"
+DEFAULT_MAX_NUM_SEQS="1"
+DEFAULT_GPU_MEM_UTIL="0.90"

presets/scout.env

@@ -0,0 +1,12 @@
+# Llama 4 Scout 默认配置
+DEFAULT_IMAGE="llama4-bi150:2026-10-26"
+DEFAULT_NAME="llama4-scout"
+DEFAULT_PORT="8091"
+DEFAULT_MODEL_HOST_DIR="/mnt/disk0/models/LLM-Research/Llama-4-Scout-17B-16E-Instruct"
+DEFAULT_SERVED_NAME="llama4-scout"
+DEFAULT_TP="8"
+DEFAULT_PP="1"
+DEFAULT_GPUS="0,1,2,3,4,5,6,7"
+DEFAULT_MAX_MODEL_LEN="4096"
+DEFAULT_MAX_NUM_SEQS="1"
+DEFAULT_GPU_MEM_UTIL="0.90"

run.sh

@@ -0,0 +1,148 @@
+#!/usr/bin/env bash
+#
+# Llama 4 Scout / Maverick 一键构建 + 启动（Iluvatar BI-V150）
+#
+#   ./run.sh scout
+#   ./run.sh maverick
+#
+# 常用覆盖项：
+#   GPUS=8,9,10,11,12,13,14,15 ./run.sh scout
+#   MODEL_HOST_DIR=/path/to/model ./run.sh maverick
+#   EXTRA_ARGS="--max-model-len 8192" ./run.sh scout
+#   EXTRA_DOCKER_ARGS="-v /mnt/disk1:/data:ro" ./run.sh scout
+#   BUILD=0 ./run.sh scout          # 使用已有镜像，不重新构建
+#   DRY_RUN=1 ./run.sh scout        # 只打印 docker run 命令
+#
+set -euo pipefail
+
+cd "$(dirname "$0")"
+
+usage() {
+  cat >&2 <<'EOF'
+用法: ./run.sh <scout|maverick>
+
+预设默认值在 presets/<name>.env 中，所有值都可以用环境变量覆盖：
+  IMAGE NAME PORT MODEL_HOST_DIR MODEL_MOUNT SERVED_NAME
+  TP PP GPUS MAX_MODEL_LEN MAX_NUM_SEQS GPU_MEM_UTIL
+  ENFORCE_EAGER EXTRA_ARGS EXTRA_DOCKER_ARGS
+  BUILD=0/1  DRY_RUN=0/1
+EOF
+}
+
+VARIANT="${1:-}"
+if [[ -z "${VARIANT}" || "${VARIANT}" == "-h" || "${VARIANT}" == "--help" ]]; then
+  usage
+  exit 2
+fi
+
+PRESET="presets/${VARIANT}.env"
+if [[ ! -f "${PRESET}" ]]; then
+  echo "ERROR: 未知预设: ${VARIANT}" >&2
+  usage
+  exit 2
+fi
+
+# shellcheck disable=SC1090
+source "${PRESET}"
+
+IMAGE="${IMAGE:-${DEFAULT_IMAGE}}"
+NAME="${NAME:-${DEFAULT_NAME}}"
+PORT="${PORT:-${DEFAULT_PORT}}"
+MODEL_HOST_DIR="${MODEL_HOST_DIR:-${DEFAULT_MODEL_HOST_DIR}}"
+MODEL_MOUNT="${MODEL_MOUNT:-/models/llama4}"
+SERVED_NAME="${SERVED_NAME:-${DEFAULT_SERVED_NAME}}"
+TP="${TP:-${DEFAULT_TP}}"
+PP="${PP:-${DEFAULT_PP}}"
+GPUS="${GPUS:-${CUDA_VISIBLE_DEVICES:-${DEFAULT_GPUS}}}"
+MAX_MODEL_LEN="${MAX_MODEL_LEN:-${DEFAULT_MAX_MODEL_LEN}}"
+MAX_NUM_SEQS="${MAX_NUM_SEQS:-${DEFAULT_MAX_NUM_SEQS}}"
+GPU_MEM_UTIL="${GPU_MEM_UTIL:-${DEFAULT_GPU_MEM_UTIL}}"
+ENFORCE_EAGER="${ENFORCE_EAGER:-1}"
+EXTRA_ARGS="${EXTRA_ARGS:-}"
+EXTRA_DOCKER_ARGS="${EXTRA_DOCKER_ARGS:-}"
+BUILD="${BUILD:-1}"
+DRY_RUN="${DRY_RUN:-0}"
+
+GPUS="${GPUS// /}"
+
+if [[ ! -f "${MODEL_HOST_DIR}/config.json" ]]; then
+  echo "ERROR: 模型目录不完整: ${MODEL_HOST_DIR}/config.json" >&2
+  echo "请先下载模型，或设置 MODEL_HOST_DIR 指向正确目录。" >&2
+  exit 1
+fi
+
+EXPECTED_GPUS=$((TP * PP))
+if [[ -n "${GPUS}" ]]; then
+  IFS=',' read -r -a GPU_LIST <<< "${GPUS}"
+  if (( ${#GPU_LIST[@]} < EXPECTED_GPUS )); then
+    echo "ERROR: TP*PP=${EXPECTED_GPUS}，但 GPUS 只给了 ${#GPU_LIST[@]} 张: ${GPUS}" >&2
+    exit 1
+  fi
+  if (( ${#GPU_LIST[@]} > EXPECTED_GPUS )); then
+    echo "WARN: TP*PP=${EXPECTED_GPUS}，但 GPUS 给了 ${#GPU_LIST[@]} 张；vLLM 只会使用其中 ${EXPECTED_GPUS} 张。" >&2
+  fi
+fi
+
+DOCKER_EXTRA=()
+if [[ -n "${EXTRA_DOCKER_ARGS}" ]]; then
+  # 按空白拆分；路径中不要带空格。
+  read -r -a DOCKER_EXTRA <<< "${EXTRA_DOCKER_ARGS}"
+fi
+
+DOCKER_RUN=(
+  docker run -d --name "${NAME}"
+  --privileged
+  --pid host
+  --network host
+  --ipc host
+  -v /dev:/dev
+  -v "${MODEL_HOST_DIR}:${MODEL_MOUNT}:ro"
+  --ulimit memlock=-1
+  --ulimit stack=67108864
+  -e CUDA_VISIBLE_DEVICES="${GPUS}"
+  -e MODEL_PATH="${MODEL_MOUNT}"
+  -e SERVED_NAME="${SERVED_NAME}"
+  -e PORT="${PORT}"
+  -e TP="${TP}"
+  -e PP="${PP}"
+  -e MAX_MODEL_LEN="${MAX_MODEL_LEN}"
+  -e MAX_NUM_SEQS="${MAX_NUM_SEQS}"
+  -e GPU_MEM_UTIL="${GPU_MEM_UTIL}"
+  -e ENFORCE_EAGER="${ENFORCE_EAGER}"
+  -e EXTRA_ARGS="${EXTRA_ARGS}"
+  "${DOCKER_EXTRA[@]}"
+  --restart unless-stopped
+  "${IMAGE}"
+)
+
+if [[ "${DRY_RUN}" == "1" ]]; then
+  printf '%q ' "${DOCKER_RUN[@]}"
+  printf '\n'
+  exit 0
+fi
+
+if [[ "${BUILD}" == "1" ]]; then
+  echo "==> build ${IMAGE}"
+  docker build -t "${IMAGE}" .
+else
+  echo "==> skip build (BUILD=0), image=${IMAGE}"
+fi
+
+echo "==> recreate ${NAME}"
+docker rm -f "${NAME}" 2>/dev/null || true
+
+echo "==> run ${VARIANT}: GPUs=${GPUS}, TP=${TP}, PP=${PP}, model=${MODEL_HOST_DIR}"
+"${DOCKER_RUN[@]}"
+
+echo "==> wait for http://127.0.0.1:${PORT}/health"
+for _ in $(seq 1 120); do
+  if curl -sf "http://127.0.0.1:${PORT}/health" >/dev/null 2>&1; then
+    echo "ready: http://127.0.0.1:${PORT}/v1/models"
+    exit 0
+  fi
+  sleep 5
+done
+
+echo "ERROR: 服务 10 分钟未就绪，查看日志：" >&2
+echo "  docker logs -f ${NAME}" >&2
+exit 1