From caddeadd15ca32d840b4eecbf4f3de0b9c272bdf Mon Sep 17 00:00:00 2001 From: xiejiahao Date: Mon, 26 Oct 2026 16:24:16 +0800 Subject: [PATCH] Unified Llama 4 Scout/Maverick BI-V150 deployment bundle --- .dockerignore | 7 + .gitea/workflows/docker-build-push.yml | 132 +++++++++++++++ .gitea/workflows/task_info.env | 3 + .gitignore | 3 + Dockerfile | 50 ++++++ README.md | 222 +++++++++++++++++++++++++ apply_patch.py | 117 +++++++++++++ docker-compose.yml | 54 ++++++ entrypoint.sh | 71 ++++++++ patches/01_fp8_quant.patch | 46 +++++ patches/02_fp8_moe.patch | 183 ++++++++++++++++++++ patches/03_llama4_chunked_attn.patch | 16 ++ patches/corexx_fp8_moe.py | 125 ++++++++++++++ presets/maverick.env | 12 ++ presets/scout.env | 12 ++ run.sh | 148 +++++++++++++++++ 16 files changed, 1201 insertions(+) create mode 100644 .dockerignore create mode 100644 .gitea/workflows/docker-build-push.yml create mode 100644 .gitea/workflows/task_info.env create mode 100644 .gitignore create mode 100644 Dockerfile create mode 100644 README.md create mode 100755 apply_patch.py create mode 100644 docker-compose.yml create mode 100755 entrypoint.sh create mode 100644 patches/01_fp8_quant.patch create mode 100644 patches/02_fp8_moe.patch create mode 100644 patches/03_llama4_chunked_attn.patch create mode 100644 patches/corexx_fp8_moe.py create mode 100644 presets/maverick.env create mode 100644 presets/scout.env create mode 100755 run.sh diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..bb437ed --- /dev/null +++ b/.dockerignore @@ -0,0 +1,7 @@ +.git +.gitea +README.md +run.sh +docker-compose.yml +presets +*.log diff --git a/.gitea/workflows/docker-build-push.yml b/.gitea/workflows/docker-build-push.yml new file mode 100644 index 0000000..9c5c285 --- /dev/null +++ b/.gitea/workflows/docker-build-push.yml @@ -0,0 +1,132 @@ +name: Docker Build and Push + +on: + push: + tags: + - "v*" + +jobs: + docker: + runs-on: amd64-ubuntu-24.04 + + steps: + - name: Clone repository + run: | + git clone "${{ gitea.server_url }}/${{ gitea.repository }}.git" . + git checkout "${{ gitea.ref_name }}" + + - name: Set image metadata + run: | + IMAGE_NAME="$(echo "${{ gitea.repository }}" | tr '[:upper:]' '[:lower:]' | tr '_' '-')" + IMAGE="${DOCKER_REGISTRY}/${DOCKER_USERNAME}/${IMAGE_NAME}:${{ gitea.ref_name }}" + + echo "IMAGE_NAME=${IMAGE_NAME}" >> "$GITEA_ENV" + echo "IMAGE=${IMAGE}" >> "$GITEA_ENV" + + - name: Load and Validate Task Info + run: | + set -a + . .gitea/workflows/task_info.env + set +a + + for name in FRAMEWORK GPU_TYPE TASK_TYPE; do + eval "value=\${${name}:-}" + if [ "$name" = "FRAMEWORK" ] && [ -z "$value" ]; then + echo "${name} is empty in .gitea/workflows/task_info.env" + exit 1 + fi + + echo "${name}=${value}" >> "$GITEA_ENV" + done + + - name: Validate Image Verify Metadata + run: | + if [ -z "${FIXED_TOKEN:-}" ]; then + echo "FIXED_TOKEN is not configured on runner" + exit 1 + fi + + if ! response="$(curl --silent --show-error --location --get 'https://modelhub.org.cn/adminApi/image-verify/validate' \ + --header "Xc-Token: ${FIXED_TOKEN}" \ + --data-urlencode "gpuType=${GPU_TYPE:-}" \ + --data-urlencode "taskType=${TASK_TYPE:-}")"; then + echo "failed to call image verify validate API" + exit 1 + fi + + VALIDATE_RESPONSE="$response" python3 - <<'PY' + import json + import os + import sys + + raw = os.environ.get("VALIDATE_RESPONSE", "") + try: + body = json.loads(raw) + except json.JSONDecodeError: + print("image verify validate API returned invalid JSON") + print(raw) + sys.exit(1) + + if body.get("code") == 0 and body.get("data") is True: + print("image verify metadata validation passed") + sys.exit(0) + + message = body.get("message") or "unknown error" + print(f"image verify metadata validation failed: {message}") + print(raw) + sys.exit(1) + PY + + - name: Login to Docker Registry + run: | + echo "$DOCKER_PASSWORD" | docker login "$DOCKER_REGISTRY" \ + -u "$DOCKER_USERNAME" \ + --password-stdin + + - name: Build Docker Image + run: | + docker build -t "$IMAGE" . + + - name: Push Docker Image + run: | + for attempt in 1 2 3; do + echo "Starting docker push attempt ${attempt}/3 for ${IMAGE}" + docker push "$IMAGE" & + PUSH_PID=$! + + while kill -0 "$PUSH_PID" 2>/dev/null; do + echo "docker push is still running at $(date -u '+%Y-%m-%dT%H:%M:%SZ')" + sleep 60 + done + + if wait "$PUSH_PID"; then + echo "docker push completed successfully" + exit 0 + fi + + echo "docker push failed on attempt ${attempt}/3" + sleep 30 + done + + echo "docker push failed after 3 attempts" + exit 1 + + - name: Notify Image Verify + run: | + if [ -z "${FIXED_TOKEN:-}" ]; then + echo "FIXED_TOKEN is not configured on runner" + exit 1 + fi + + curl --silent --show-error --fail-with-body --location --request POST 'https://modelhub.org.cn//adminApi/image-verify' \ + --header "Xc-Token: ${FIXED_TOKEN}" \ + --header 'Content-Type: application/json' \ + --data-raw "{ + \"framework\": \"${FRAMEWORK}\", + \"gpuType\": \"${GPU_TYPE}\", + \"imageUrl\": \"${IMAGE}\", + \"taskType\": \"${TASK_TYPE}\", + \"createBy\": \"${{ gitea.actor }}\", + \"repoUrl\": \"${{ gitea.server_url }}/${{ gitea.repository }}\", + \"tag\": \"${{ github.ref_name }}\" + }" diff --git a/.gitea/workflows/task_info.env b/.gitea/workflows/task_info.env new file mode 100644 index 0000000..339ca74 --- /dev/null +++ b/.gitea/workflows/task_info.env @@ -0,0 +1,3 @@ +FRAMEWORK=vllm +GPU_TYPE=Iluvatar_bi-150 +TASK_TYPE=text-generation diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..873c238 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +*.log +__pycache__/ +*.pyc diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..fb5b7b1 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,50 @@ +# Llama 4 Scout / Maverick on Iluvatar BI-V150 +# +# 一个镜像同时支持： +# - Llama-4-Scout-17B-16E-Instruct（BF16，TP8，8 卡） +# - Llama-4-Maverick-17B-128E-Instruct-FP8（FP8，TP8+PP2，16 卡） +# +# 模型权重不打进镜像，运行时按 run.sh 的 scout / maverick 预设挂载。 +# +# 构建： +# docker build -t llama4-bi150:2026-10-26 . +# +# 运行： +# ./run.sh scout +# ./run.sh maverick +# +# 补丁是针对下面这个精确基础镜像生成的；换基础镜像需要重新生成 diff。 +# registry.iluvatar.com.cn:10443/customer/sz/vllm0.17.0-4.4.0-x86:v7 +# (vLLM 0.17.0+corex.20260612070235) + +FROM registry.iluvatar.com.cn:10443/customer/sz/vllm0.17.0-4.4.0-x86:v7 + +ARG IMAGE_VERSION=2026-10-26 + +LABEL org.opencontainers.image.title="llama4-bi150" \ + org.opencontainers.image.description="Llama 4 Scout / Maverick on Iluvatar BI-V150 with vLLM 0.17.0+corex" \ + org.opencontainers.image.version="${IMAGE_VERSION}" + +COPY patches /opt/llama4/patches +COPY apply_patch.py /opt/llama4/apply_patch.py +COPY entrypoint.sh /usr/local/bin/llama4-entrypoint.sh + +RUN chmod +x /usr/local/bin/llama4-entrypoint.sh \ + && python3 /opt/llama4/apply_patch.py + +ENV MODEL_PATH=/models/llama4 \ + SERVED_NAME=llama4 \ + HOST=0.0.0.0 \ + PORT=8091 \ + TP=8 \ + PP=1 \ + MAX_MODEL_LEN=4096 \ + MAX_NUM_SEQS=1 \ + GPU_MEM_UTIL=0.90 \ + ENFORCE_EAGER=1 \ + PYTHONUNBUFFERED=1 + +WORKDIR /opt/llama4 +EXPOSE 8091 8092 + +ENTRYPOINT ["/usr/local/bin/llama4-entrypoint.sh"] diff --git a/README.md b/README.md new file mode 100644 index 0000000..15f0046 --- /dev/null +++ b/README.md @@ -0,0 +1,222 @@ +# Llama 4 Scout / Maverick on Iluvatar BI-V150 + +在 16×Iluvatar BI-V150 32GB 上运行 Llama 4 Scout 与 Maverick 的统一 Docker 部署仓库。 + +- 基础镜像：`registry.iluvatar.com.cn:10443/customer/sz/vllm0.17.0-4.4.0-x86:v7`（vLLM `0.17.0+corex`） +- 一个镜像同时支持 Scout 和 Maverick；模型权重不打进镜像，运行时只读挂载 +- 两个模型共用 Llama 4 chunked-attention 补丁；Maverick 额外使用 FP8 量化 / FP8 MoE 补丁 +- CI 使用 ModelHub 标准 Gitea Actions：推送 `v*` 标签后构建、推送镜像并提交审核 + +| 模型 | 精度 | 并行 | GPU 数 | 默认端口 | +|---|---|---|---|---| +| `LLM-Research/Llama-4-Scout-17B-16E-Instruct` | BF16 | TP8 | 8 | 8091 | +| `LLM-Research/Llama-4-Maverick-17B-128E-Instruct-FP8` | FP8 | TP8 + PP2 | 16 | 8092 | + +--- + +## 1. 目录内容 + +```text +llama4-bi150-deploy/ +├── Dockerfile +├── entrypoint.sh +├── apply_patch.py # 幂等应用全部 Llama 4 补丁 +├── patches/ +│ ├── 01_fp8_quant.patch # Maverick FP8 量化 +│ ├── 02_fp8_moe.patch # Maverick FP8 MoE +│ ├── 03_llama4_chunked_attn.patch +│ └── corexx_fp8_moe.py # 逐 expert 反量化 BF16 的 FP8 MoE 回退 +├── presets/ +│ ├── scout.env +│ └── maverick.env +├── run.sh # 构建 + 启动 + 等 /health +├── docker-compose.yml # 可选，按 profile 启动 +├── .gitea/workflows/ +│ ├── docker-build-push.yml +│ └── task_info.env +└── README.md +``` + +--- + +## 2. 构建与启动 + +### 2.1 脚本（推荐） + +```bash +# Scout：8 卡，8091 +./run.sh scout + +# Maverick：16 卡，8092 +./run.sh maverick +``` + +`run.sh` 会先 `docker build`，再删除同名容器、启动新容器，并等待 `/health` 最多 10 分钟。 + +### 2.2 指定 GPU 和模型目录 + +所有预设值都可以用环境变量覆盖： + +```bash +# Scout 使用 8-15 号卡 +GPUS=8,9,10,11,12,13,14,15 ./run.sh scout + +# Maverick 使用 0-15 号卡 +GPUS=0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15 ./run.sh maverick + +# 模型放在其他目录 +MODEL_HOST_DIR=/mnt/disk1/models/Llama-4-Scout-17B-16E-Instruct ./run.sh scout + +# 额外挂载一个目录 +EXTRA_DOCKER_ARGS="-v /mnt/disk1:/data:ro" ./run.sh scout + +# 传递额外 vLLM 参数 +EXTRA_ARGS="--max-model-len 8192" ./run.sh scout + +# 已有镜像，不重新构建 +BUILD=0 IMAGE=registry.example.com/llama4-bi150:v1.0.0 ./run.sh scout + +# 只打印 docker run 命令，便于检查 +DRY_RUN=1 ./run.sh scout +``` + +### 2.3 Docker Compose（可选） + +```bash +docker compose --profile scout up -d --build +docker compose --profile maverick up -d --build + +# 覆盖模型目录 / GPU +SCOUT_MODEL_HOST_DIR=/mnt/disk1/models/scout SCOUT_GPUS=8,9,10,11,12,13,14,15 \ + docker compose --profile scout up -d --build +``` + +--- + +## 3. 可配置项 + +| 变量 | Scout 默认 | Maverick 默认 | 说明 | +|---|---|---|---| +| `IMAGE` | `llama4-bi150:2026-10-26` | 同左 | 镜像名 | +| `NAME` | `llama4-scout` | `llama4-maverick` | 容器名 | +| `PORT` | `8091` | `8092` | 宿主机端口 | +| `MODEL_HOST_DIR` | `.../Llama-4-Scout-17B-16E-Instruct` | `.../Llama-4-Maverick-17B-128E-Instruct-FP8` | 宿主机模型目录 | +| `MODEL_MOUNT` | `/models/llama4` | 同左 | 容器内挂载点 | +| `SERVED_NAME` | `llama4-scout` | `llama4-maverick` | OpenAI API 模型名 | +| `TP` | `8` | `8` | 张量并行 | +| `PP` | `1` | `2` | 流水线并行 | +| `GPUS` | `0,...,7` | `0,...,15` | 传给 `CUDA_VISIBLE_DEVICES` | +| `MAX_MODEL_LEN` | `4096` | `4096` | 最大上下文 | +| `MAX_NUM_SEQS` | `1` | `1` | 最大并发 | +| `GPU_MEM_UTIL` | `0.90` | `0.90` | 显存利用率 | +| `ENFORCE_EAGER` | `1` | `1` | 是否 `--enforce-eager` | +| `EXTRA_ARGS` | 空 | 空 | 额外 vLLM 参数 | +| `EXTRA_DOCKER_ARGS` | 空 | 空 | 额外 docker 参数，如 `-v` | +| `BUILD` | `1` | `1` | `0` 表示跳过构建 | +| `DRY_RUN` | `0` | `0` | `1` 只打印命令 | + +`GPUS` 也可以直接用 `CUDA_VISIBLE_DEVICES` 设置。`run.sh` 会检查 GPU 数是否少于 `TP*PP`，少于时直接报错。 + +--- + +## 4. 验证 + +```bash +# 健康检查 +curl -s http://127.0.0.1:8091/health +curl -s http://127.0.0.1:8092/health + +# 模型列表 +curl -s http://127.0.0.1:8091/v1/models | python3 -m json.tool +curl -s http://127.0.0.1:8092/v1/models | python3 -m json.tool + +# 文本 +curl -s http://127.0.0.1:8091/v1/chat/completions \ + -H 'Content-Type: application/json' \ + -d '{"model":"llama4-scout","messages":[{"role":"user","content":"你好"}],"max_tokens":64}' +``` + +Maverick 图片输入测试： + +```python +from PIL import Image +import base64, io, requests + +img = Image.new("RGB", (336, 336), (220, 20, 20)) +buf = io.BytesIO(); img.save(buf, format="PNG") +b64 = base64.b64encode(buf.getvalue()).decode() + +payload = { + "model": "llama4-maverick", + "messages": [{"role": "user", "content": [ + {"type": "text", "text": "这张图片主要是什么颜色？只回答颜色。"}, + {"type": "image_url", "image_url": {"url": "data:image/png;base64," + b64}}, + ]}], + "max_tokens": 32, + "temperature": 0, +} +print(requests.post("http://127.0.0.1:8092/v1/chat/completions", + json=payload, timeout=300).json()["choices"][0]["message"]["content"]) +``` + +首次使用统一镜像时，建议把 Scout 和 Maverick 都跑一遍，确认 FP8 补丁没有影响 Scout 的 BF16 路径。 + +--- + +## 5. CI / ModelHub 发布 + +仓库已包含 ModelHub 标准 CI： + +```text +.gitea/workflows/docker-build-push.yml +.gitea/workflows/task_info.env +``` + +触发方式： + +```bash +git tag v1.0.0 +git push origin v1.0.0 +``` + +CI 会： + +1. 在 `amd64-ubuntu-24.04` runner 上 clone 仓库； +2. 校验 `task_info.env`； +3. `docker build`； +4. 推送 `${DOCKER_REGISTRY}/${DOCKER_USERNAME}/llama-4-bi150:v1.0.0`； +5. 调 ModelHub `image-verify` 接口提交审核。 + +`task_info.env`： + +```bash +FRAMEWORK=vllm +GPU_TYPE=Iluvatar_bi-150 +TASK_TYPE=text-generation +``` + +Runner 需要配置以下环境变量（不是仓库 secret）： + +```text +DOCKER_REGISTRY +DOCKER_USERNAME +DOCKER_PASSWORD +FIXED_TOKEN +``` + +注意事项： + +- Runner 必须能拉取 `registry.iluvatar.com.cn:10443` 的基础镜像；如果该仓库需要登录，请在 runner 上预登录，或自行给 workflow 增加基础镜像 registry 的 login step。 +- CI 只负责构建和推送，不负责 GPU / 模型挂载；GPU 和挂载通过 `run.sh` 或 `docker compose` 设置。 +- CI 构建机不需要 GPU，但需要足够的磁盘空间。 +- 如果仓库改名，CI 生成的镜像名会跟着变。 + +--- + +## 6. 限制与注意事项 + +- Scout 需要 8 张卡（TP8）；Maverick 需要 16 张卡（TP8+PP2）。16 卡机器上两者不能同时运行。 +- 当前默认 `MAX_MODEL_LEN=4096`、`MAX_NUM_SEQS=1`，适合功能验证；生产并发需要先调整显存/上下文。 +- 补丁是针对当前精确基础镜像生成的。更换基础镜像后不要直接复用，需要重新生成 diff 并在目标环境验证。 +- `--privileged --pid host -v /dev:/dev` 是 Iluvatar 运行时要求；只给 `/dev/iluvatar*` 可能报 `cudaGetDeviceCount() ... Error 1001`。 +- 模型目录只读挂载，容器不会修改宿主机模型文件。 diff --git a/apply_patch.py b/apply_patch.py new file mode 100755 index 0000000..b94f23e --- /dev/null +++ b/apply_patch.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python3 +"""Apply the CoreX vLLM patches needed by Llama 4 Scout / Maverick. + +The patches in ./patches were generated against the exact base image +`registry.iluvatar.com.cn:10443/customer/sz/vllm0.17.0-4.4.0-x86:v7` +(vLLM 0.17.0+corex.20260612070235). + +Scout needs only the chunked-attention patch. Maverick additionally needs the +FP8 quantization / FP8 MoE patches and the `corexx_fp8_moe` fallback module. +One image contains the union of both patch sets; the FP8 paths are only used +when the mounted model is FP8. + +The script is idempotent: it checks for a marker in each target file and skips +patches that are already applied. +""" + +from __future__ import annotations + +import ast +import os +import pathlib +import shutil +import subprocess +import sys + +PATCH_DIR = pathlib.Path( + os.environ.get("LLAMA4_PATCH_DIR", "/opt/llama4/patches") +) + +# LLAMA4_PKG_ROOT is only needed for local testing against a copy of the +# vendor files. In the image it is derived from the installed vllm package. +_pkg_env = os.environ.get("LLAMA4_PKG_ROOT") +if _pkg_env: + PKG = pathlib.Path(_pkg_env) +else: + import vllm + + PKG = pathlib.Path(vllm.__file__).resolve().parent.parent + +# patch file -> (target file relative to the package root, applied marker) +TARGETS = { + "01_fp8_quant.patch": ( + "vllm/_custom_ops.py", + "COREXX_FP8_QUANT_PATCH", + ), + "02_fp8_moe.patch": ( + "vllm/model_executor/layers/fused_moe/fused_moe.py", + "COREXX_FP8_MOE_PATCH", + ), + "03_llama4_chunked_attn.patch": ( + "vllm/v1/attention/backends/utils.py", + "seq_lens_np=seqlens_k_local", + ), +} + + +def apply_one(patch_name: str, target_rel: str, marker: str) -> None: + target = PKG / target_rel + if not target.exists(): + sys.exit(f"target file not found: {target}") + if marker in target.read_text(): + print(f"[skip] {target} (already patched)") + return + + patch_file = PATCH_DIR / patch_name + if not patch_file.exists(): + sys.exit(f"missing patch file: {patch_file}") + + result = subprocess.run( + [ + "patch", + "-p1", + "--forward", + "--batch", + "--no-backup-if-mismatch", + "-d", + str(PKG), + "-i", + str(patch_file), + ], + text=True, + capture_output=True, + ) + if result.stdout: + print(result.stdout, end="") + if result.returncode != 0: + if result.stderr: + print(result.stderr, file=sys.stderr) + sys.exit(f"patch failed: {patch_name}") + if marker not in target.read_text(): + sys.exit(f"patch {patch_name} applied but marker not found in {target}") + print(f"[ok] {target}") + + +def main() -> None: + for patch_name, (target_rel, marker) in TARGETS.items(): + apply_one(patch_name, target_rel, marker) + + src = PATCH_DIR / "corexx_fp8_moe.py" + dst = PKG / "vllm/model_executor/layers/fused_moe/corexx_fp8_moe.py" + if not src.exists(): + sys.exit(f"missing module: {src}") + shutil.copy2(src, dst) + print(f"[copy] {src} -> {dst}") + + for path in ( + PKG / "vllm/_custom_ops.py", + PKG / "vllm/model_executor/layers/fused_moe/fused_moe.py", + dst, + PKG / "vllm/v1/attention/backends/utils.py", + ): + ast.parse(path.read_text()) + print("all patches applied; syntax OK") + + +if __name__ == "__main__": + main() diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..cdee6d5 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,54 @@ +x-llama4-common: &llama4-common + build: + context: . + dockerfile: Dockerfile + image: llama4-bi150:${TAG:-2026-10-26} + network_mode: host + ipc: host + privileged: true + pid: host + ulimits: + memlock: -1 + stack: 67108864 + restart: unless-stopped + +services: + scout: + <<: *llama4-common + profiles: ["scout"] + container_name: llama4-scout + volumes: + - /dev:/dev + - ${SCOUT_MODEL_HOST_DIR:-/mnt/disk0/models/LLM-Research/Llama-4-Scout-17B-16E-Instruct}:/models/llama4:ro + environment: + MODEL_PATH: /models/llama4 + SERVED_NAME: llama4-scout + PORT: "8091" + TP: "8" + PP: "1" + MAX_MODEL_LEN: "4096" + MAX_NUM_SEQS: "1" + GPU_MEM_UTIL: "0.90" + ENFORCE_EAGER: "1" + CUDA_VISIBLE_DEVICES: "${SCOUT_GPUS:-0,1,2,3,4,5,6,7}" + EXTRA_ARGS: "${SCOUT_EXTRA_ARGS:-}" + + maverick: + <<: *llama4-common + profiles: ["maverick"] + container_name: llama4-maverick + volumes: + - /dev:/dev + - ${MAVERICK_MODEL_HOST_DIR:-/mnt/disk0/models/LLM-Research/Llama-4-Maverick-17B-128E-Instruct-FP8}:/models/llama4:ro + environment: + MODEL_PATH: /models/llama4 + SERVED_NAME: llama4-maverick + PORT: "8092" + TP: "8" + PP: "2" + MAX_MODEL_LEN: "4096" + MAX_NUM_SEQS: "1" + GPU_MEM_UTIL: "0.90" + ENFORCE_EAGER: "1" + CUDA_VISIBLE_DEVICES: "${MAVERICK_GPUS:-0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15}" + EXTRA_ARGS: "${MAVERICK_EXTRA_ARGS:-}" diff --git a/entrypoint.sh b/entrypoint.sh new file mode 100755 index 0000000..0fa192f --- /dev/null +++ b/entrypoint.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash +# +# Llama 4 Scout / Maverick 统一容器入口： +# 1. 幂等应用 CoreX vLLM 的 Llama 4 补丁（Scout 只用 chunked attention， +# Maverick 额外使用 FP8 补丁；补丁脚本会按 marker 自动跳过已应用项） +# 2. 检查模型目录 +# 3. exec vLLM（信号可直接传给 vLLM） +# +set -euo pipefail + +PATCH_SCRIPT="${LLAMA4_PATCH_SCRIPT:-/opt/llama4/apply_patch.py}" +VLLM_BIN="${VLLM_BIN:-vllm}" + +python3 "${PATCH_SCRIPT}" + +MODEL_PATH="${MODEL_PATH:-/models/llama4}" +SERVED_NAME="${SERVED_NAME:-llama4}" +HOST="${HOST:-0.0.0.0}" +PORT="${PORT:-8091}" +TP="${TP:-8}" +PP="${PP:-1}" +MAX_MODEL_LEN="${MAX_MODEL_LEN:-4096}" +MAX_NUM_SEQS="${MAX_NUM_SEQS:-1}" +GPU_MEM_UTIL="${GPU_MEM_UTIL:-0.90}" +ENFORCE_EAGER="${ENFORCE_EAGER:-1}" +EXTRA_ARGS="${EXTRA_ARGS:-}" + +if [[ ! -f "${MODEL_PATH}/config.json" ]]; then + echo "ERROR: 模型目录不完整：${MODEL_PATH}/config.json 不存在" >&2 + echo "请确认已挂载模型，例如：" >&2 + echo " -v /mnt/disk0/models/LLM-Research/Llama-4-Scout-17B-16E-Instruct:/models/llama4:ro" >&2 + echo " -v /mnt/disk0/models/LLM-Research/Llama-4-Maverick-17B-128E-Instruct-FP8:/models/llama4:ro" >&2 + exit 1 +fi + +PP_ARGS=() +if (( PP > 1 )); then + PP_ARGS=(--pipeline-parallel-size "${PP}") +fi + +EAGER_ARGS=() +if [[ "${ENFORCE_EAGER}" == "1" ]]; then + EAGER_ARGS=(--enforce-eager) +fi + +echo "============================================================" +echo "Llama 4 (BI-V150 unified image)" +echo " model : ${MODEL_PATH}" +echo " served name: ${SERVED_NAME}" +echo " listen : ${HOST}:${PORT}" +echo " TP / PP : ${TP} / ${PP} (total $((TP * PP)) GPUs)" +echo " max len : ${MAX_MODEL_LEN}" +echo " max seqs : ${MAX_NUM_SEQS}" +echo " gpu util : ${GPU_MEM_UTIL}" +echo " eager : ${ENFORCE_EAGER}" +echo " visible : ${CUDA_VISIBLE_DEVICES:-all}" +echo " extra : ${EXTRA_ARGS}" +echo "============================================================" + +# shellcheck disable=SC2086 +exec "${VLLM_BIN}" serve "${MODEL_PATH}" \ + --host "${HOST}" \ + --port "${PORT}" \ + --served-model-name "${SERVED_NAME}" \ + --tensor-parallel-size "${TP}" \ + "${PP_ARGS[@]}" \ + --max-model-len "${MAX_MODEL_LEN}" \ + --max-num-seqs "${MAX_NUM_SEQS}" \ + --gpu-memory-utilization "${GPU_MEM_UTIL}" \ + "${EAGER_ARGS[@]}" \ + ${EXTRA_ARGS} diff --git a/patches/01_fp8_quant.patch b/patches/01_fp8_quant.patch new file mode 100644 index 0000000..318d9d8 --- /dev/null +++ b/patches/01_fp8_quant.patch @@ -0,0 +1,46 @@ +--- a/vllm/_custom_ops.py ++++ b/vllm/_custom_ops.py +@@ -1933,14 +1933,28 @@ + if scale is None: + if use_per_token_if_dynamic: + scale = torch.empty((shape[0], 1), device=input.device, dtype=torch.float32) +- torch.ops._C.dynamic_per_token_scaled_fp8_quant( +- output, input, scale, scale_ub +- ) ++ # COREXX_FP8_QUANT_PATCH: native fallback (vLLM _C op missing on CoreX) ++ m = input.shape[0] ++ s = input.abs().amax(dim=1, keepdim=True).float().clamp_min(1e-12) / 448.0 ++ if scale_ub is not None: ++ ub = torch.as_tensor(scale_ub, device=s.device, dtype=s.dtype) ++ s = torch.minimum(s, ub) ++ scale[:m].copy_(s) ++ if shape[0] > m: ++ scale[m:].fill_(1.0) ++ output[m:].zero_() ++ output[:m].copy_((input.float() / s).to(out_dtype)) + else: + scale = torch.empty(1, device=input.device, dtype=torch.float32) +- torch.ops._C.dynamic_scaled_fp8_quant(output, input, scale) ++ # COREXX_FP8_QUANT_PATCH: native fallback ++ amax = input.abs().amax().float().clamp_min(1e-12) ++ scale.fill_(amax / 448.0) ++ output.copy_((input.float() / scale).to(out_dtype)) + else: +- torch.ops._C.static_scaled_fp8_quant(output, input, scale, group_shape) ++ # COREXX_FP8_QUANT_PATCH: native fallback ++ if group_shape is not None and tuple(group_shape) not in ((-1, -1), (-1, 1), (1, -1)): ++ raise NotImplementedError(f"native fp8 static quant: unsupported group_shape {group_shape}") ++ output.copy_((input.float() / scale).to(out_dtype)) + + return output, scale + +@@ -3824,6 +3838,9 @@ + mul_routed_weight, + top_k, + config['BLOCK_SIZE_M'], ++ # COREXX_FP8_MOE_PATCH: pass quant scales to ixformer kernel ++ w_scale=B_scale, ++ a_scale=A_scale, + bias=bias + ) + diff --git a/patches/02_fp8_moe.patch b/patches/02_fp8_moe.patch new file mode 100644 index 0000000..815a235 --- /dev/null +++ b/patches/02_fp8_moe.patch @@ -0,0 +1,183 @@ +--- a/vllm/model_executor/layers/fused_moe/fused_moe.py ++++ b/vllm/model_executor/layers/fused_moe/fused_moe.py +@@ -1645,7 +1645,38 @@ + # check constraints + if use_fp8_w8a8 or use_int8_w8a8 or use_int8_w8a16 or use_int4_w4a16 or w1_scale or \ + w2_scale or w1_zp or w2_zp or a1_scale or a2_scale: +- raise ValueError("Quantized MoE is not supported") ++ # COREXX_FP8_MOE_PATCH: generic fused_experts_impl supports quantized MoE ++ res = fused_experts_impl( ++ hidden_states=hidden_states, ++ w1=w1, ++ w2=w2, ++ topk_weights=topk_weights, ++ topk_ids=topk_ids, ++ inplace=False, ++ activation=activation, ++ apply_router_weight_on_input=apply_router_weight_on_input, ++ use_fp8_w8a8=use_fp8_w8a8, ++ use_int8_w8a8=use_int8_w8a8, ++ use_int8_w8a16=use_int8_w8a16, ++ use_int4_w4a16=use_int4_w4a16, ++ ocp_mx_scheme=ocp_mx_scheme, ++ per_channel_quant=per_channel_quant, ++ global_num_experts=global_num_experts, ++ expert_map=expert_map, ++ w1_scale=w1_scale, ++ w2_scale=w2_scale, ++ w1_zp=w1_zp, ++ w2_zp=w2_zp, ++ a1_scale=a1_scale, ++ a2_scale=a2_scale, ++ block_shape=block_shape, ++ w1_bias=w1_bias, ++ w2_bias=w2_bias, ++ ) ++ if output is not None: ++ output.copy_(res) ++ return output ++ return res + + attn_metadata = get_forward_context().attn_metadata + use_ep = expert_map is not None +@@ -1912,8 +1943,32 @@ + w1_bias: torch.Tensor | None = None, + w2_bias: torch.Tensor | None = None, + ) -> torch.Tensor: ++ # COREXX_FP8_MOE_PATCH: dedicated per-expert BF16 fallback for FP8 MoE ++ if hidden_states.dtype in ( ++ torch.float8_e4m3fn, torch.float8_e5m2, ++ torch.float8_e4m3fnuz, torch.float8_e5m2fnuz, ++ ): ++ from vllm.model_executor.layers.fused_moe.corexx_fp8_moe import corexx_fp8_moe ++ return corexx_fp8_moe( ++ hidden_states=hidden_states, ++ w1=w1, ++ w2=w2, ++ topk_weights=topk_weights, ++ topk_ids=topk_ids, ++ activation=activation, ++ apply_router_weight_on_input=apply_router_weight_on_input, ++ global_num_experts=global_num_experts, ++ expert_map=expert_map, ++ w1_scale=w1_scale, ++ w2_scale=w2_scale, ++ a1_scale=a1_scale, ++ a2_scale=a2_scale, ++ block_shape=block_shape, ++ w1_bias=w1_bias, ++ w2_bias=w2_bias, ++ ) ++ + # Convert string activation to enum for internal use +- activation_enum = MoEActivation.from_str(activation) + + # Check constraints. + if use_int4_w4a16: +@@ -1937,7 +1992,12 @@ + assert hidden_states.is_contiguous(), "Hidden_states must be contiguous" + assert w1.stride(-1) == 1, "Stride of last dimension must be 1" + assert w2.stride(-1) == 1, "Stride of last dimension must be 1" +- assert hidden_states.dtype in [torch.float32, torch.float16, torch.bfloat16] ++ # COREXX_FP8_MOE_PATCH: modular kernel may pass pre-quantized FP8 input ++ assert hidden_states.dtype in [ ++ torch.float32, torch.float16, torch.bfloat16, ++ torch.float8_e4m3fn, torch.float8_e5m2, ++ torch.float8_e4m3fnuz, torch.float8_e5m2fnuz, ++ ] + + num_tokens = hidden_states.size(0) + E, N, _ = w1.size() +@@ -1949,6 +2009,9 @@ + # https://github.com/vllm-project/vllm/issues/5938 + CHUNK_SIZE = envs.VLLM_FUSED_MOE_CHUNK_SIZE + M = min(num_tokens, CHUNK_SIZE) ++ # COREXX_FP8_MOE_PATCH: FP8 input still needs BF16 compute buffers/output ++ io_dtype = hidden_states.dtype if hidden_states.dtype in ( ++ torch.float32, torch.float16, torch.bfloat16) else torch.bfloat16 + + config_dtype = _get_config_dtype_str( + use_fp8_w8a8=use_fp8_w8a8, +@@ -1982,7 +2045,7 @@ + cache13 = torch.empty( + M * top_k_num * max(N, K), + device=hidden_states.device, +- dtype=hidden_states.dtype, ++ dtype=io_dtype, + ) + intermediate_cache1 = cache13[: M * top_k_num * N].view(M, top_k_num, N) + intermediate_cache3 = cache13[: M * top_k_num * K].view(M, top_k_num, K) +@@ -1994,19 +2057,22 @@ + intermediate_cache2 = torch.empty( + (M * top_k_num, activation_out_dim), + device=hidden_states.device, +- dtype=hidden_states.dtype, ++ dtype=io_dtype, + ) + +- if hidden_states.dtype == torch.bfloat16: ++ if io_dtype == torch.bfloat16: + compute_type = tl.bfloat16 +- elif hidden_states.dtype == torch.float16: ++ elif io_dtype == torch.float16: + compute_type = tl.float16 +- elif hidden_states.dtype == torch.float32: ++ elif io_dtype == torch.float32: + compute_type = tl.float32 + else: +- raise ValueError(f"Unsupported compute_type: {hidden_states.dtype}") ++ raise ValueError(f"Unsupported compute_type: {io_dtype}") + +- out_hidden_states = hidden_states if inplace else torch.empty_like(hidden_states) ++ if inplace and io_dtype == hidden_states.dtype: ++ out_hidden_states = hidden_states ++ else: ++ out_hidden_states = torch.empty_like(hidden_states, dtype=io_dtype) + + if ocp_mx_scheme is not None: + # TODO: On platforms for which `current_platform.supports_mx()` is True +@@ -2064,14 +2130,24 @@ + + curr_topk_ids = topk_ids[begin_chunk_idx:end_chunk_idx] + curr_topk_weights = topk_weights[begin_chunk_idx:end_chunk_idx] +- qcurr_hidden_states, a1q_scale = moe_kernel_quantize_input( +- A=curr_hidden_states, +- A_scale=a1_scale, +- quant_dtype=quant_dtype, +- per_act_token_quant=per_channel_quant, +- block_shape=block_shape, +- ocp_mx_scheme=ocp_mx_scheme, +- ) ++ # COREXX_FP8_MOE_PATCH: use pre-quantized FP8 input as-is ++ if curr_hidden_states.dtype in ( ++ torch.float8_e4m3fn, torch.float8_e5m2, ++ torch.float8_e4m3fnuz, torch.float8_e5m2fnuz, ++ ): ++ qcurr_hidden_states = curr_hidden_states ++ a1q_scale = a1_scale ++ if a1q_scale is None: ++ raise ValueError("pre-quantized MoE input requires a1_scale") ++ else: ++ qcurr_hidden_states, a1q_scale = moe_kernel_quantize_input( ++ A=curr_hidden_states, ++ A_scale=a1_scale, ++ quant_dtype=quant_dtype, ++ per_act_token_quant=per_channel_quant, ++ block_shape=block_shape, ++ ocp_mx_scheme=ocp_mx_scheme, ++ ) + + # # SPARSITY_FACTOR is a heuristic margin ensuring tokens_in_chunk * top_k + # # activates only a small fraction of total experts +@@ -2305,8 +2381,9 @@ + self.quant_config.w2_scale, + self.quant_config.w1_zp, + self.quant_config.w2_zp, +- self.quant_config.a1_scale, +- self.quant_config.a2_scale, ++ # COREXX_FP8_MOE_PATCH: prefer per-call dynamic scales ++ a1q_scale if a1q_scale is not None else self.quant_config.a1_scale, ++ a2_scale if a2_scale is not None else self.quant_config.a2_scale, + self.quant_config.block_shape, + self.quant_config.w1_bias, + self.quant_config.w2_bias, diff --git a/patches/03_llama4_chunked_attn.patch b/patches/03_llama4_chunked_attn.patch new file mode 100644 index 0000000..6ffa683 --- /dev/null +++ b/patches/03_llama4_chunked_attn.patch @@ -0,0 +1,16 @@ +--- a/vllm/v1/attention/backends/utils.py ++++ b/vllm/v1/attention/backends/utils.py +@@ -346,7 +346,13 @@ + return CommonAttentionMetadata( + query_start_loc_cpu=query_start_loc_cpu, + query_start_loc=query_start_loc_cpu.to(device=device, non_blocking=True), ++ key_start_loc=torch.from_numpy( ++ np.concatenate( ++ (np.zeros(1, dtype=np.int32), np.cumsum(seqlens_k_local, dtype=np.int32)) ++ ) ++ ).to(device=device, non_blocking=True), + seq_lens=seq_lens_cpu.to(device=device, non_blocking=True), ++ seq_lens_np=seqlens_k_local, + num_reqs=len(seq_lens_cpu), + num_actual_tokens=common_attn_metadata.num_actual_tokens, + max_query_len=seqlens_q_local.max(), diff --git a/patches/corexx_fp8_moe.py b/patches/corexx_fp8_moe.py new file mode 100644 index 0000000..b436e1b --- /dev/null +++ b/patches/corexx_fp8_moe.py @@ -0,0 +1,125 @@ +"""CoreX fallback for FP8 W8A8 MoE on Iluvatar BI-V150. + +The CoreX ixformer MoE kernels are W16A16 only and torch._scaled_mm supports +only tensor-wise scales, so per-token/per-channel FP8 MoE cannot use them. +This module dequantizes one expert at a time (each expert shard is only a few +hundred MB on a TP rank), runs BF16 matmuls, and scatters the result. +""" +from __future__ import annotations + +import torch +import torch.nn.functional as F + +from vllm.logger import init_logger + +logger = init_logger(__name__) +_FP8_DTYPES = ( + torch.float8_e4m3fn, + torch.float8_e5m2, + torch.float8_e4m3fnuz, + torch.float8_e5m2fnuz, +) +_LOGGED = False + + +def _dequant_weight(w: torch.Tensor, scale: torch.Tensor | None) -> torch.Tensor: + """w: [rows, cols] FP8, scale: per-output-row (len rows) or scalar.""" + if scale is None: + return w.to(torch.bfloat16) + s = scale.reshape(-1) + if s.numel() == 1: + return (w.float() * s[0]).to(torch.bfloat16) + if s.numel() != w.shape[0]: + # try to broadcast a [1, rows] style scale + s = s.reshape(w.shape[0], -1)[:, 0] + return (w.float() * s.reshape(-1, 1)).to(torch.bfloat16) + + +def corexx_fp8_moe( + hidden_states: torch.Tensor, + w1: torch.Tensor, + w2: torch.Tensor, + topk_weights: torch.Tensor, + topk_ids: torch.Tensor, + activation, + apply_router_weight_on_input: bool, + global_num_experts: int, + expert_map: torch.Tensor | None, + w1_scale: torch.Tensor | None, + w2_scale: torch.Tensor | None, + a1_scale: torch.Tensor | None, + a2_scale: torch.Tensor | None, + block_shape, + w1_bias: torch.Tensor | None, + w2_bias: torch.Tensor | None, +) -> torch.Tensor: + global _LOGGED + if expert_map is not None: + raise NotImplementedError("CoreX FP8 MoE fallback does not support expert_map/EP") + if block_shape is not None: + raise NotImplementedError("CoreX FP8 MoE fallback does not support block FP8") + if not _LOGGED: + logger.info( + "CoreX FP8 MoE fallback: hidden=%s %s a1_scale=%s w1=%s w1_scale=%s w2=%s w2_scale=%s topk=%s", + tuple(hidden_states.shape), hidden_states.dtype, + None if a1_scale is None else tuple(a1_scale.shape), + tuple(w1.shape), None if w1_scale is None else tuple(w1_scale.shape), + tuple(w2.shape), None if w2_scale is None else tuple(w2_scale.shape), + tuple(topk_ids.shape), + ) + _LOGGED = True + + if hidden_states.dtype in _FP8_DTYPES: + if a1_scale is None: + raise ValueError("FP8 MoE input requires a1_scale") + A = (hidden_states.float() * a1_scale.reshape(-1, 1)).to(torch.bfloat16) + else: + A = hidden_states + + E = w1.shape[0] + N = w1.shape[1] + K = w1.shape[2] + I = w2.shape[2] + assert N == 2 * I, f"expected w1 N=2*I, got {N} vs {I}" + + act_name = getattr(activation, "value", activation) + + M = A.shape[0] + out = torch.zeros((M, K), dtype=torch.bfloat16, device=A.device) + topk = topk_ids.shape[1] + + # COREXX_FP8_MOE_PATCH: only touch experts that actually received tokens + active_experts = torch.unique(topk_ids).tolist() + for e in active_experts: + if e < 0 or e >= E: + continue + # rows of A routed to expert e (global id == local id when expert_map is None) + for k in range(topk): + idx = (topk_ids[:, k] == e).nonzero(as_tuple=False).flatten() + if idx.numel() == 0: + continue + x = A.index_select(0, idx) + if apply_router_weight_on_input: + x = x * topk_weights[idx, k].to(x.dtype).unsqueeze(1) + w1_e = _dequant_weight(w1[e], None if w1_scale is None else w1_scale[e]) + h = x @ w1_e.t() + if w1_bias is not None: + h = h + w1_bias[e] + gate = h[:, :I] + up = h[:, I:N] + if act_name == "silu": + act = F.silu(gate) * up + elif act_name == "gelu": + act = F.gelu(gate) * up + else: + raise NotImplementedError(f"CoreX FP8 MoE activation {act_name}") + w2_e = _dequant_weight(w2[e], None if w2_scale is None else w2_scale[e]) + y = act @ w2_e.t() + if w2_bias is not None: + y = y + w2_bias[e] + if not apply_router_weight_on_input: + y = y * topk_weights[idx, k].to(y.dtype).unsqueeze(1) + out.index_add_(0, idx, y) + del w1_e, w2_e, h, act, y + + return out diff --git a/presets/maverick.env b/presets/maverick.env new file mode 100644 index 0000000..00b3e95 --- /dev/null +++ b/presets/maverick.env @@ -0,0 +1,12 @@ +# Llama 4 Maverick FP8 默认配置 +DEFAULT_IMAGE="llama4-bi150:2026-10-26" +DEFAULT_NAME="llama4-maverick" +DEFAULT_PORT="8092" +DEFAULT_MODEL_HOST_DIR="/mnt/disk0/models/LLM-Research/Llama-4-Maverick-17B-128E-Instruct-FP8" +DEFAULT_SERVED_NAME="llama4-maverick" +DEFAULT_TP="8" +DEFAULT_PP="2" +DEFAULT_GPUS="0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15" +DEFAULT_MAX_MODEL_LEN="4096" +DEFAULT_MAX_NUM_SEQS="1" +DEFAULT_GPU_MEM_UTIL="0.90" diff --git a/presets/scout.env b/presets/scout.env new file mode 100644 index 0000000..6fae6af --- /dev/null +++ b/presets/scout.env @@ -0,0 +1,12 @@ +# Llama 4 Scout 默认配置 +DEFAULT_IMAGE="llama4-bi150:2026-10-26" +DEFAULT_NAME="llama4-scout" +DEFAULT_PORT="8091" +DEFAULT_MODEL_HOST_DIR="/mnt/disk0/models/LLM-Research/Llama-4-Scout-17B-16E-Instruct" +DEFAULT_SERVED_NAME="llama4-scout" +DEFAULT_TP="8" +DEFAULT_PP="1" +DEFAULT_GPUS="0,1,2,3,4,5,6,7" +DEFAULT_MAX_MODEL_LEN="4096" +DEFAULT_MAX_NUM_SEQS="1" +DEFAULT_GPU_MEM_UTIL="0.90" diff --git a/run.sh b/run.sh new file mode 100755 index 0000000..737f6e6 --- /dev/null +++ b/run.sh @@ -0,0 +1,148 @@ +#!/usr/bin/env bash +# +# Llama 4 Scout / Maverick 一键构建 + 启动（Iluvatar BI-V150） +# +# ./run.sh scout +# ./run.sh maverick +# +# 常用覆盖项： +# GPUS=8,9,10,11,12,13,14,15 ./run.sh scout +# MODEL_HOST_DIR=/path/to/model ./run.sh maverick +# EXTRA_ARGS="--max-model-len 8192" ./run.sh scout +# EXTRA_DOCKER_ARGS="-v /mnt/disk1:/data:ro" ./run.sh scout +# BUILD=0 ./run.sh scout # 使用已有镜像，不重新构建 +# DRY_RUN=1 ./run.sh scout # 只打印 docker run 命令 +# +set -euo pipefail + +cd "$(dirname "$0")" + +usage() { + cat >&2 <<'EOF' +用法: ./run.sh + +预设默认值在 presets/.env 中，所有值都可以用环境变量覆盖： + IMAGE NAME PORT MODEL_HOST_DIR MODEL_MOUNT SERVED_NAME + TP PP GPUS MAX_MODEL_LEN MAX_NUM_SEQS GPU_MEM_UTIL + ENFORCE_EAGER EXTRA_ARGS EXTRA_DOCKER_ARGS + BUILD=0/1 DRY_RUN=0/1 +EOF +} + +VARIANT="${1:-}" +if [[ -z "${VARIANT}" || "${VARIANT}" == "-h" || "${VARIANT}" == "--help" ]]; then + usage + exit 2 +fi + +PRESET="presets/${VARIANT}.env" +if [[ ! -f "${PRESET}" ]]; then + echo "ERROR: 未知预设: ${VARIANT}" >&2 + usage + exit 2 +fi + +# shellcheck disable=SC1090 +source "${PRESET}" + +IMAGE="${IMAGE:-${DEFAULT_IMAGE}}" +NAME="${NAME:-${DEFAULT_NAME}}" +PORT="${PORT:-${DEFAULT_PORT}}" +MODEL_HOST_DIR="${MODEL_HOST_DIR:-${DEFAULT_MODEL_HOST_DIR}}" +MODEL_MOUNT="${MODEL_MOUNT:-/models/llama4}" +SERVED_NAME="${SERVED_NAME:-${DEFAULT_SERVED_NAME}}" +TP="${TP:-${DEFAULT_TP}}" +PP="${PP:-${DEFAULT_PP}}" +GPUS="${GPUS:-${CUDA_VISIBLE_DEVICES:-${DEFAULT_GPUS}}}" +MAX_MODEL_LEN="${MAX_MODEL_LEN:-${DEFAULT_MAX_MODEL_LEN}}" +MAX_NUM_SEQS="${MAX_NUM_SEQS:-${DEFAULT_MAX_NUM_SEQS}}" +GPU_MEM_UTIL="${GPU_MEM_UTIL:-${DEFAULT_GPU_MEM_UTIL}}" +ENFORCE_EAGER="${ENFORCE_EAGER:-1}" +EXTRA_ARGS="${EXTRA_ARGS:-}" +EXTRA_DOCKER_ARGS="${EXTRA_DOCKER_ARGS:-}" +BUILD="${BUILD:-1}" +DRY_RUN="${DRY_RUN:-0}" + +GPUS="${GPUS// /}" + +if [[ ! -f "${MODEL_HOST_DIR}/config.json" ]]; then + echo "ERROR: 模型目录不完整: ${MODEL_HOST_DIR}/config.json" >&2 + echo "请先下载模型，或设置 MODEL_HOST_DIR 指向正确目录。" >&2 + exit 1 +fi + +EXPECTED_GPUS=$((TP * PP)) +if [[ -n "${GPUS}" ]]; then + IFS=',' read -r -a GPU_LIST <<< "${GPUS}" + if (( ${#GPU_LIST[@]} < EXPECTED_GPUS )); then + echo "ERROR: TP*PP=${EXPECTED_GPUS}，但 GPUS 只给了 ${#GPU_LIST[@]} 张: ${GPUS}" >&2 + exit 1 + fi + if (( ${#GPU_LIST[@]} > EXPECTED_GPUS )); then + echo "WARN: TP*PP=${EXPECTED_GPUS}，但 GPUS 给了 ${#GPU_LIST[@]} 张；vLLM 只会使用其中 ${EXPECTED_GPUS} 张。" >&2 + fi +fi + +DOCKER_EXTRA=() +if [[ -n "${EXTRA_DOCKER_ARGS}" ]]; then + # 按空白拆分；路径中不要带空格。 + read -r -a DOCKER_EXTRA <<< "${EXTRA_DOCKER_ARGS}" +fi + +DOCKER_RUN=( + docker run -d --name "${NAME}" + --privileged + --pid host + --network host + --ipc host + -v /dev:/dev + -v "${MODEL_HOST_DIR}:${MODEL_MOUNT}:ro" + --ulimit memlock=-1 + --ulimit stack=67108864 + -e CUDA_VISIBLE_DEVICES="${GPUS}" + -e MODEL_PATH="${MODEL_MOUNT}" + -e SERVED_NAME="${SERVED_NAME}" + -e PORT="${PORT}" + -e TP="${TP}" + -e PP="${PP}" + -e MAX_MODEL_LEN="${MAX_MODEL_LEN}" + -e MAX_NUM_SEQS="${MAX_NUM_SEQS}" + -e GPU_MEM_UTIL="${GPU_MEM_UTIL}" + -e ENFORCE_EAGER="${ENFORCE_EAGER}" + -e EXTRA_ARGS="${EXTRA_ARGS}" + "${DOCKER_EXTRA[@]}" + --restart unless-stopped + "${IMAGE}" +) + +if [[ "${DRY_RUN}" == "1" ]]; then + printf '%q ' "${DOCKER_RUN[@]}" + printf '\n' + exit 0 +fi + +if [[ "${BUILD}" == "1" ]]; then + echo "==> build ${IMAGE}" + docker build -t "${IMAGE}" . +else + echo "==> skip build (BUILD=0), image=${IMAGE}" +fi + +echo "==> recreate ${NAME}" +docker rm -f "${NAME}" 2>/dev/null || true + +echo "==> run ${VARIANT}: GPUs=${GPUS}, TP=${TP}, PP=${PP}, model=${MODEL_HOST_DIR}" +"${DOCKER_RUN[@]}" + +echo "==> wait for http://127.0.0.1:${PORT}/health" +for _ in $(seq 1 120); do + if curl -sf "http://127.0.0.1:${PORT}/health" >/dev/null 2>&1; then + echo "ready: http://127.0.0.1:${PORT}/v1/models" + exit 0 + fi + sleep 5 +done + +echo "ERROR: 服务 10 分钟未就绪，查看日志：" >&2 +echo " docker logs -f ${NAME}" >&2 +exit 1