diff --git a/.dockerignore b/.dockerignore index bb437ed..988de73 100644 --- a/.dockerignore +++ b/.dockerignore @@ -2,6 +2,6 @@ .gitea README.md run.sh +verify.sh docker-compose.yml -presets *.log diff --git a/Dockerfile b/Dockerfile index fb5b7b1..2d593eb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,14 +4,15 @@ # - Llama-4-Scout-17B-16E-Instruct(BF16,TP8,8 卡) # - Llama-4-Maverick-17B-128E-Instruct-FP8(FP8,TP8+PP2,16 卡) # -# 模型权重不打进镜像,运行时按 run.sh 的 scout / maverick 预设挂载。 +# 模型权重不打进镜像,运行时挂载模型目录;容器启动后读取 config.json +# 自动判断 Scout / Maverick,并选择对应的 TP / PP / 端口。 # # 构建: # docker build -t llama4-bi150:2026-10-26 . # # 运行: -# ./run.sh scout -# ./run.sh maverick +# ./run.sh /mnt/disk0/models/LLM-Research/Llama-4-Scout-17B-16E-Instruct +# ./run.sh /mnt/disk0/models/LLM-Research/Llama-4-Maverick-17B-128E-Instruct-FP8 # # 补丁是针对下面这个精确基础镜像生成的;换基础镜像需要重新生成 diff。 # registry.iluvatar.com.cn:10443/customer/sz/vllm0.17.0-4.4.0-x86:v7 @@ -27,17 +28,16 @@ LABEL org.opencontainers.image.title="llama4-bi150" \ COPY patches /opt/llama4/patches COPY apply_patch.py /opt/llama4/apply_patch.py +COPY detect_model.py /opt/llama4/detect_model.py COPY entrypoint.sh /usr/local/bin/llama4-entrypoint.sh RUN chmod +x /usr/local/bin/llama4-entrypoint.sh \ + && chmod +x /opt/llama4/detect_model.py \ && python3 /opt/llama4/apply_patch.py ENV MODEL_PATH=/models/llama4 \ - SERVED_NAME=llama4 \ + AUTO_DETECT=1 \ HOST=0.0.0.0 \ - PORT=8091 \ - TP=8 \ - PP=1 \ MAX_MODEL_LEN=4096 \ MAX_NUM_SEQS=1 \ GPU_MEM_UTIL=0.90 \ diff --git a/README.md b/README.md index 3215f54..dbf771d 100644 --- a/README.md +++ b/README.md @@ -3,17 +3,17 @@ 在 16×Iluvatar BI-V150 32GB 上运行 Llama 4 Scout 与 Maverick 的统一 Docker 部署仓库。 > 本仓库合并自原先独立的 `llama4-scout-deploy` 和 `llama4-maverick-deploy`, -> 用一个镜像 + 两个运行预设替代两套重复的 Dockerfile。 +> 用一个镜像 + 自动模型检测替代两套重复的 Dockerfile 和手动 profile。 - 基础镜像:`registry.iluvatar.com.cn:10443/customer/sz/vllm0.17.0-4.4.0-x86:v7`(vLLM `0.17.0+corex`) - 一个镜像同时支持 Scout 和 Maverick;模型权重不打进镜像,运行时只读挂载 - 两个模型共用 Llama 4 chunked-attention 补丁;Maverick 额外使用 FP8 量化 / FP8 MoE 补丁 - CI 使用 ModelHub 标准 Gitea Actions:推送 `v*` 标签后构建、推送镜像并提交审核 -| 模型 | 精度 | 并行 | GPU 数 | 默认端口 | +| 模型 | 精度 | 自动选择 | GPU 数 | 默认端口 | |---|---|---|---|---| -| `LLM-Research/Llama-4-Scout-17B-16E-Instruct` | BF16 | TP8 | 8 | 8091 | -| `LLM-Research/Llama-4-Maverick-17B-128E-Instruct-FP8` | FP8 | TP8 + PP2 | 16 | 8092 | +| `Llama-4-Scout-17B-16E-Instruct` | BF16 | TP8 / PP1 | 8 | 8091 | +| `Llama-4-Maverick-17B-128E-Instruct-FP8` | FP8 | TP8 / PP2 | 16 | 8092 | --- @@ -24,149 +24,202 @@ llama4-bi150-deploy/ ├── Dockerfile ├── entrypoint.sh ├── apply_patch.py # 幂等应用全部 Llama 4 补丁 +├── detect_model.py # 从 config.json 判断 Scout / Maverick +├── verify.sh # 服务验证脚本 ├── patches/ │ ├── 01_fp8_quant.patch # Maverick FP8 量化 │ ├── 02_fp8_moe.patch # Maverick FP8 MoE │ ├── 03_llama4_chunked_attn.patch │ └── corexx_fp8_moe.py # 逐 expert 反量化 BF16 的 FP8 MoE 回退 -├── presets/ -│ ├── scout.env -│ └── maverick.env -├── run.sh # 构建 + 启动 + 等 /health -├── docker-compose.yml # 可选,按 profile 启动 +├── run.sh # 自动识别模型 + 构建 + 启动 + 等 /health +├── docker-compose.yml # 可选 ├── .gitea/workflows/ │ ├── docker-build-push.yml │ └── task_info.env └── README.md ``` +BI-150 上代码放在: + +```text +/root/xjh/llama4-bi150-deploy +``` + +该路径是 `ugly_hopper` 容器内的 `/root/xjh`,直通宿主机。 + --- -## 2. 构建与启动 +## 2. 快速开始 -### 2.1 脚本(推荐) +不需要传 `scout` / `maverick` 参数,`run.sh` 会读取模型目录里的 `config.json` 自动判断。 ```bash -# Scout:8 卡,8091 -./run.sh scout +cd /root/xjh/llama4-bi150-deploy -# Maverick:16 卡,8092 -./run.sh maverick +# Scout +./run.sh /mnt/disk0/models/LLM-Research/Llama-4-Scout-17B-16E-Instruct + +# Maverick +./run.sh /mnt/disk0/models/LLM-Research/Llama-4-Maverick-17B-128E-Instruct-FP8 + +# 也可以用环境变量 +MODEL_HOST_DIR=/mnt/disk0/models/LLM-Research/Llama-4-Maverick-17B-128E-Instruct-FP8 ./run.sh + +# 不传参数时,依次尝试两个默认模型目录 +./run.sh ``` -`run.sh` 会先 `docker build`,再删除同名容器、启动新容器,并等待 `/health` 最多 10 分钟。 +启动后会打印检测结果,例如: -### 2.2 指定 GPU 和模型目录 +```text +==> detected maverick: Llama 4 Maverick 17B-128E Instruct FP8 (TP8+PP2, 16 GPUs) + model : /mnt/disk0/models/LLM-Research/Llama-4-Maverick-17B-128E-Instruct-FP8 + TP/PP : 8/2 (16 GPUs), port=8092, name=llama4-maverick + GPUs : 0,1,2,...,15 +``` -所有预设值都可以用环境变量覆盖: +### 2.1 自动检测规则 + +`detect_model.py` 读取 `config.json`: + +- `model_type` 必须是 `llama4`; +- `text_config.num_local_experts == 16` → **Scout**,TP8 / PP1 / 8091; +- `text_config.num_local_experts == 128` → **Maverick**,TP8 / PP2 / 8092; +- 字段缺失时用目录名里的 `scout` / `maverick` 兜底; +- 其他情况直接报错,不会用错误的 TP/PP 启动。 + +容器启动时 `entrypoint.sh` 也会再检测一次。也就是说: + +- 用 `run.sh` 启动:宿主侧先检测,决定 GPU 数、容器名、健康检查端口; +- 直接 `docker run` 挂载模型:容器内也会自动检测,按对应 TP/PP/端口启动; +- 显式设置的 `TP` / `PP` / `PORT` / `SERVED_NAME` 会覆盖自动检测; +- 设置 `AUTO_DETECT=0` 可关闭容器内检测;未显式设置的项仍使用 `run.sh` 在宿主侧检测出的默认值。 + +### 2.2 指定 GPU + +默认使用前 `TP*PP` 张卡:Scout 用 `0-7`,Maverick 用 `0-15`。可以覆盖: ```bash -# Scout 使用 8-15 号卡 -GPUS=8,9,10,11,12,13,14,15 ./run.sh scout +# Scout 使用 8-15 +GPUS=8,9,10,11,12,13,14,15 \ + ./run.sh /mnt/disk0/models/LLM-Research/Llama-4-Scout-17B-16E-Instruct -# Maverick 使用 0-15 号卡 -GPUS=0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15 ./run.sh maverick +# Maverick 使用 0-15(默认) +GPUS=0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15 \ + ./run.sh /mnt/disk0/models/LLM-Research/Llama-4-Maverick-17B-128E-Instruct-FP8 +``` -# 模型放在其他目录 -MODEL_HOST_DIR=/mnt/disk1/models/Llama-4-Scout-17B-16E-Instruct ./run.sh scout +`run.sh` 会检查 GPU 数是否少于 `TP*PP`;少于时直接报错。 -# 额外挂载一个目录 -EXTRA_DOCKER_ARGS="-v /mnt/disk1:/data:ro" ./run.sh scout +### 2.3 额外挂载 / 额外 vLLM 参数 -# 传递额外 vLLM 参数 -EXTRA_ARGS="--max-model-len 8192" ./run.sh scout +```bash +EXTRA_DOCKER_ARGS="-v /mnt/disk1:/data:ro" ./run.sh /path/to/model +EXTRA_ARGS="--max-model-len 8192" ./run.sh /path/to/model # 已有镜像,不重新构建 -BUILD=0 IMAGE=registry.example.com/llama4-bi150:v1.0.0 ./run.sh scout +BUILD=0 IMAGE=registry.example.com/llama4-bi150:v1.0.0 ./run.sh /path/to/model -# 只打印 docker run 命令,便于检查 -DRY_RUN=1 ./run.sh scout +# 只打印 docker run 命令 +DRY_RUN=1 ./run.sh /path/to/model ``` -### 2.3 Docker Compose(可选) +### 2.4 Docker Compose(可选) + +Compose 使用单个 `llama4` service,同样由 entrypoint 自动检测模型: ```bash -docker compose --profile scout up -d --build -docker compose --profile maverick up -d --build +# Scout +MODEL_HOST_DIR=/mnt/disk0/models/LLM-Research/Llama-4-Scout-17B-16E-Instruct \ + docker compose up -d --build -# 覆盖模型目录 / GPU -SCOUT_MODEL_HOST_DIR=/mnt/disk1/models/scout SCOUT_GPUS=8,9,10,11,12,13,14,15 \ - docker compose --profile scout up -d --build +# Maverick +MODEL_HOST_DIR=/mnt/disk0/models/LLM-Research/Llama-4-Maverick-17B-128E-Instruct-FP8 \ + docker compose up -d --build -# 覆盖镜像 tag / 额外 vLLM 参数 -TAG=v1.0.0 SCOUT_EXTRA_ARGS="--max-model-len 8192" \ - docker compose --profile scout up -d --build +# 指定 GPU / 端口 / 额外参数 +MODEL_HOST_DIR=/mnt/disk0/models/... \ +GPUS=8,9,10,11,12,13,14,15 PORT=8095 \ + docker compose up -d --build ``` --- ## 3. 可配置项 -| 变量 | Scout 默认 | Maverick 默认 | 说明 | -|---|---|---|---| -| `IMAGE` | `llama4-bi150:2026-10-26` | 同左 | 镜像名 | -| `NAME` | `llama4-scout` | `llama4-maverick` | 容器名 | -| `PORT` | `8091` | `8092` | 宿主机端口 | -| `MODEL_HOST_DIR` | `.../Llama-4-Scout-17B-16E-Instruct` | `.../Llama-4-Maverick-17B-128E-Instruct-FP8` | 宿主机模型目录 | -| `MODEL_MOUNT` | `/models/llama4` | 同左 | 容器内挂载点 | -| `SERVED_NAME` | `llama4-scout` | `llama4-maverick` | OpenAI API 模型名 | -| `TP` | `8` | `8` | 张量并行 | -| `PP` | `1` | `2` | 流水线并行 | -| `GPUS` | `0,...,7` | `0,...,15` | 传给 `CUDA_VISIBLE_DEVICES` | -| `MAX_MODEL_LEN` | `4096` | `4096` | 最大上下文 | -| `MAX_NUM_SEQS` | `1` | `1` | 最大并发 | -| `GPU_MEM_UTIL` | `0.90` | `0.90` | 显存利用率 | -| `ENFORCE_EAGER` | `1` | `1` | 是否 `--enforce-eager` | -| `EXTRA_ARGS` | 空 | 空 | 额外 vLLM 参数 | -| `EXTRA_DOCKER_ARGS` | 空 | 空 | 额外 docker 参数,如 `-v` | -| `BUILD` | `1` | `1` | `0` 表示跳过构建 | -| `DRY_RUN` | `0` | `0` | `1` 只打印命令 | +| 变量 | 默认 | 说明 | +|---|---|---| +| `MODEL_HOST_DIR` | 自动尝试两个默认目录 | 宿主机模型目录 | +| `MODEL_MOUNT` | `/models/llama4` | 容器内挂载点 | +| `IMAGE` | `llama4-bi150:2026-10-26` | 镜像名 | +| `NAME` | `llama4-` | 容器名 | +| `PORT` | Scout 8091 / Maverick 8092 | 宿主机端口 | +| `SERVED_NAME` | Scout `llama4-scout` / Maverick `llama4-maverick` | OpenAI API 模型名 | +| `TP` | Scout 8 / Maverick 8 | 张量并行 | +| `PP` | Scout 1 / Maverick 2 | 流水线并行 | +| `GPUS` | 前 `TP*PP` 张卡 | `CUDA_VISIBLE_DEVICES` | +| `MAX_MODEL_LEN` | `4096` | 最大上下文 | +| `MAX_NUM_SEQS` | `1` | 最大并发 | +| `GPU_MEM_UTIL` | `0.90` | 显存利用率 | +| `ENFORCE_EAGER` | `1` | 是否 `--enforce-eager` | +| `AUTO_DETECT` | `1` | 容器内是否按 config 自动检测 | +| `EXTRA_ARGS` | 空 | 额外 vLLM 参数 | +| `EXTRA_DOCKER_ARGS` | 空 | 额外 docker 参数,如 `-v` | +| `BUILD` | `1` | `0` 表示跳过构建 | +| `DRY_RUN` | `0` | `1` 只打印命令 | -`GPUS` 也可以直接用 `CUDA_VISIBLE_DEVICES` 设置。`run.sh` 会检查 GPU 数是否少于 `TP*PP`,少于时直接报错。 +`TP` / `PP` / `PORT` / `SERVED_NAME` 显式设置时优先于自动检测。 --- -## 4. 验证 +## 4. 验证脚本 + +`verify.sh` 用于检查已经启动的服务: ```bash -# 健康检查 -curl -s http://127.0.0.1:8091/health -curl -s http://127.0.0.1:8092/health - -# 模型列表 -curl -s http://127.0.0.1:8091/v1/models | python3 -m json.tool -curl -s http://127.0.0.1:8092/v1/models | python3 -m json.tool - -# 文本 -curl -s http://127.0.0.1:8091/v1/chat/completions \ - -H 'Content-Type: application/json' \ - -d '{"model":"llama4-scout","messages":[{"role":"user","content":"你好"}],"max_tokens":64}' +./verify.sh ``` -Maverick 图片输入测试: +它会自动查找运行中的 `llama4-bi150` 容器,读取模型挂载、端口和 served name,然后检查: -```python -from PIL import Image -import base64, io, requests +1. `/health` +2. `/v1/models` +3. `/v1/chat/completions`(中文问答) -img = Image.new("RGB", (336, 336), (220, 20, 20)) -buf = io.BytesIO(); img.save(buf, format="PNG") -b64 = base64.b64encode(buf.getvalue()).decode() +常用用法: -payload = { - "model": "llama4-maverick", - "messages": [{"role": "user", "content": [ - {"type": "text", "text": "这张图片主要是什么颜色?只回答颜色。"}, - {"type": "image_url", "image_url": {"url": "data:image/png;base64," + b64}}, - ]}], - "max_tokens": 32, - "temperature": 0, -} -print(requests.post("http://127.0.0.1:8092/v1/chat/completions", - json=payload, timeout=300).json()["choices"][0]["message"]["content"]) +```bash +# 指定模型目录(推荐,能准确判断 Scout/Maverick) +./verify.sh --model-dir /mnt/disk0/models/LLM-Research/Llama-4-Maverick-17B-128E-Instruct-FP8 + +# 指定容器 +./verify.sh --container llama4-maverick + +# 额外检查图片识别(Maverick) +./verify.sh --container llama4-maverick --with-image + +# 额外检查简单推理(期望回答 27) +./verify.sh --container llama4-scout --with-reasoning + +# 直接指定服务地址 +./verify.sh --base-url http://127.0.0.1:8092 --served-name llama4-maverick ``` -首次使用统一镜像时,建议把 Scout 和 Maverick 都跑一遍,确认 FP8 补丁没有影响 Scout 的 BF16 路径。 +主要参数: + +| 参数 | 说明 | +|---|---| +| `--model-dir PATH` | 模型目录,用于自动判断 Scout/Maverick | +| `--container NAME` | 容器名;默认自动查找 | +| `--port PORT` | 服务端口;默认从模型/容器推断 | +| `--served-name NAME` | API 模型名;默认从模型/容器推断 | +| `--base-url URL` | 直接指定服务地址 | +| `--timeout SEC` | 等待/请求超时,默认 180 | +| `--with-reasoning` | 额外跑推理题 | +| `--with-image` | 额外跑图片识别(仅 Maverick) | +| `--skip-chat` | 只检查 health 和 models | + +脚本只依赖 `bash`、`curl`、`python3`;不需要 `jq`。返回码为 0 表示全部通过。 --- @@ -204,9 +257,7 @@ TASK_TYPE=text-generation Maverick 支持图片输入;如果 ModelHub 要求把图文任务单独归类,请把 `TASK_TYPE` 改成平台支持的值。`image-verify/validate` 会在构建前校验,填错会直接失败。 -Runner 需要在 Gitea 实例或仓库中注册,并带有 `amd64-ubuntu-24.04` label。 - -Runner 需要配置以下环境变量(不是仓库 secret): +Runner 需要在 Gitea 实例或仓库中注册,并带有 `amd64-ubuntu-24.04` label。Runner 需要配置以下环境变量(不是仓库 secret): ```text DOCKER_REGISTRY @@ -232,3 +283,4 @@ FIXED_TOKEN - 补丁是针对当前精确基础镜像生成的。更换基础镜像后不要直接复用,需要重新生成 diff 并在目标环境验证。 - `--privileged --pid host -v /dev:/dev` 是 Iluvatar 运行时要求;只给 `/dev/iluvatar*` 可能报 `cudaGetDeviceCount() ... Error 1001`。 - 模型目录只读挂载,容器不会修改宿主机模型文件。 +- 统一镜像首次使用时,建议 Scout 和 Maverick 都各跑一次 `verify.sh`;尤其是 Scout,因为镜像里也包含 FP8 MoE 补丁,需要确认共享的 `fused_moe.py` 修改没有影响 BF16 路径。 diff --git a/detect_model.py b/detect_model.py new file mode 100755 index 0000000..6359712 --- /dev/null +++ b/detect_model.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python3 +"""Detect a Llama 4 variant from its config.json. + +The unified image can serve both: + + - Llama-4-Scout-17B-16E-Instruct (16 local experts, BF16) -> TP8, PP1, 8 GPUs + - Llama-4-Maverick-17B-128E-Instruct-FP8 (128 local experts, FP8) -> TP8, PP2, 16 GPUs + +The detected values are printed as shell assignments by default, so run.sh and +entrypoint.sh can `eval` them. `--json` is used by verify.sh. + +Detection is based on `text_config.num_local_experts`, with a model-directory +name fallback if the field is missing. Unsupported configs fail loudly rather +than silently starting with the wrong TP/PP. +""" + +from __future__ import annotations + +import argparse +import json +import pathlib +import shlex +import sys + +PROFILES = { + "scout": { + "tp": 8, + "pp": 1, + "port": 8091, + "served_name": "llama4-scout", + "gpu_count": 8, + "description": "Llama 4 Scout 17B-16E Instruct (BF16, TP8, 8 GPUs)", + }, + "maverick": { + "tp": 8, + "pp": 2, + "port": 8092, + "served_name": "llama4-maverick", + "gpu_count": 16, + "description": "Llama 4 Maverick 17B-128E Instruct FP8 (TP8+PP2, 16 GPUs)", + }, +} + + +def load_config(model_dir: str) -> dict: + path = pathlib.Path(model_dir) / "config.json" + if not path.is_file(): + raise ValueError(f"config.json not found: {path}") + try: + return json.loads(path.read_text()) + except json.JSONDecodeError as exc: + raise ValueError(f"invalid config.json: {exc}") from exc + + +def detect(model_dir: str) -> dict: + cfg = load_config(model_dir) + model_type = str(cfg.get("model_type", "")) + if model_type != "llama4": + raise ValueError( + f"unsupported model_type={model_type!r}; expected 'llama4'" + ) + + text = cfg.get("text_config") or {} + experts = text.get("num_local_experts") + if experts is None: + experts = text.get("num_experts") + + if experts is not None: + experts = int(experts) + if experts >= 64: + kind = "maverick" + elif experts <= 32: + kind = "scout" + else: + raise ValueError( + f"cannot determine Llama 4 variant: num_local_experts={experts}; " + "expected 16 (Scout) or 128 (Maverick)" + ) + detection = f"num_local_experts={experts}" + else: + name = pathlib.Path(model_dir).name.lower() + if "maverick" in name: + kind = "maverick" + elif "scout" in name: + kind = "scout" + else: + raise ValueError( + "cannot determine Llama 4 variant: text_config.num_local_experts " + "is missing and the directory name does not contain scout/maverick" + ) + detection = "directory name" + + profile = dict(PROFILES[kind]) + profile.update( + { + "kind": kind, + "model_type": model_type, + "architecture": (cfg.get("architectures") or [""])[0], + "num_local_experts": experts, + "quantized": bool(cfg.get("quantization_config")), + "detection": detection, + "model_dir": str(pathlib.Path(model_dir).resolve()), + } + ) + return profile + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Detect Llama 4 Scout/Maverick from config.json" + ) + parser.add_argument("model_dir", help="directory containing config.json") + parser.add_argument( + "--json", + action="store_true", + help="print JSON instead of shell assignments", + ) + args = parser.parse_args() + + try: + profile = detect(args.model_dir) + except ValueError as exc: + print(f"ERROR: {exc}", file=sys.stderr) + return 2 + + if args.json: + print(json.dumps(profile, ensure_ascii=False, indent=2)) + return 0 + + values = { + "DETECTED_MODEL_KIND": profile["kind"], + "DETECTED_TP": profile["tp"], + "DETECTED_PP": profile["pp"], + "DETECTED_PORT": profile["port"], + "DETECTED_SERVED_NAME": profile["served_name"], + "DETECTED_GPU_COUNT": profile["gpu_count"], + "DETECTED_DESCRIPTION": profile["description"], + } + for key, value in values.items(): + print(f"{key}={shlex.quote(str(value))}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/docker-compose.yml b/docker-compose.yml index cdee6d5..91a8886 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,54 +1,46 @@ -x-llama4-common: &llama4-common - build: - context: . - dockerfile: Dockerfile - image: llama4-bi150:${TAG:-2026-10-26} - network_mode: host - ipc: host - privileged: true - pid: host - ulimits: - memlock: -1 - stack: 67108864 - restart: unless-stopped - services: - scout: - <<: *llama4-common - profiles: ["scout"] - container_name: llama4-scout - volumes: - - /dev:/dev - - ${SCOUT_MODEL_HOST_DIR:-/mnt/disk0/models/LLM-Research/Llama-4-Scout-17B-16E-Instruct}:/models/llama4:ro - environment: - MODEL_PATH: /models/llama4 - SERVED_NAME: llama4-scout - PORT: "8091" - TP: "8" - PP: "1" - MAX_MODEL_LEN: "4096" - MAX_NUM_SEQS: "1" - GPU_MEM_UTIL: "0.90" - ENFORCE_EAGER: "1" - CUDA_VISIBLE_DEVICES: "${SCOUT_GPUS:-0,1,2,3,4,5,6,7}" - EXTRA_ARGS: "${SCOUT_EXTRA_ARGS:-}" + llama4: + build: + context: . + dockerfile: Dockerfile + image: llama4-bi150:${TAG:-2026-10-26} + container_name: llama4 + + # vLLM 多卡 NCCL / 端口直通 + network_mode: host + ipc: host + + # Iluvatar 运行时要求 + privileged: true + pid: host - maverick: - <<: *llama4-common - profiles: ["maverick"] - container_name: llama4-maverick volumes: - /dev:/dev - - ${MAVERICK_MODEL_HOST_DIR:-/mnt/disk0/models/LLM-Research/Llama-4-Maverick-17B-128E-Instruct-FP8}:/models/llama4:ro + - ${MODEL_HOST_DIR:-/mnt/disk0/models/LLM-Research/Llama-4-Scout-17B-16E-Instruct}:/models/llama4:ro + environment: MODEL_PATH: /models/llama4 - SERVED_NAME: llama4-maverick - PORT: "8092" - TP: "8" - PP: "2" - MAX_MODEL_LEN: "4096" - MAX_NUM_SEQS: "1" - GPU_MEM_UTIL: "0.90" - ENFORCE_EAGER: "1" - CUDA_VISIBLE_DEVICES: "${MAVERICK_GPUS:-0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15}" - EXTRA_ARGS: "${MAVERICK_EXTRA_ARGS:-}" + AUTO_DETECT: "1" + HOST: 0.0.0.0 + + # 默认暴露全部 16 张卡;entrypoint 会根据模型选择实际 TP*PP。 + CUDA_VISIBLE_DEVICES: "${GPUS:-0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15}" + + # 留空时由 entrypoint 根据 config.json 自动判断。 + # 如果需要手动覆盖,取消注释并填值。 + # TP: "${TP:-}" + # PP: "${PP:-}" + # PORT: "${PORT:-}" + # SERVED_NAME: "${SERVED_NAME:-}" + + MAX_MODEL_LEN: "${MAX_MODEL_LEN:-4096}" + MAX_NUM_SEQS: "${MAX_NUM_SEQS:-1}" + GPU_MEM_UTIL: "${GPU_MEM_UTIL:-0.90}" + ENFORCE_EAGER: "${ENFORCE_EAGER:-1}" + EXTRA_ARGS: "${EXTRA_ARGS:-}" + + ulimits: + memlock: -1 + stack: 67108864 + + restart: unless-stopped diff --git a/entrypoint.sh b/entrypoint.sh index 0fa192f..38d9c20 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -1,29 +1,24 @@ #!/usr/bin/env bash # -# Llama 4 Scout / Maverick 统一容器入口: -# 1. 幂等应用 CoreX vLLM 的 Llama 4 补丁(Scout 只用 chunked attention, -# Maverick 额外使用 FP8 补丁;补丁脚本会按 marker 自动跳过已应用项) -# 2. 检查模型目录 -# 3. exec vLLM(信号可直接传给 vLLM) +# Llama 4 统一容器入口: +# 1. 幂等应用 CoreX vLLM 的 Llama 4 补丁 +# 2. 读取 /models/llama4/config.json,自动判断 Scout / Maverick +# 3. 选择对应的 TP / PP / 端口 / served name,再启动 vLLM +# +# AUTO_DETECT=1(默认)时根据模型 config 自动选择;显式设置的 +# TP / PP / PORT / SERVED_NAME 会覆盖检测结果。AUTO_DETECT=0 可完全手动。 # set -euo pipefail PATCH_SCRIPT="${LLAMA4_PATCH_SCRIPT:-/opt/llama4/apply_patch.py}" +DETECT_SCRIPT="${LLAMA4_DETECT_SCRIPT:-/opt/llama4/detect_model.py}" VLLM_BIN="${VLLM_BIN:-vllm}" python3 "${PATCH_SCRIPT}" MODEL_PATH="${MODEL_PATH:-/models/llama4}" -SERVED_NAME="${SERVED_NAME:-llama4}" -HOST="${HOST:-0.0.0.0}" -PORT="${PORT:-8091}" -TP="${TP:-8}" -PP="${PP:-1}" -MAX_MODEL_LEN="${MAX_MODEL_LEN:-4096}" -MAX_NUM_SEQS="${MAX_NUM_SEQS:-1}" -GPU_MEM_UTIL="${GPU_MEM_UTIL:-0.90}" -ENFORCE_EAGER="${ENFORCE_EAGER:-1}" -EXTRA_ARGS="${EXTRA_ARGS:-}" +AUTO_DETECT="${AUTO_DETECT:-1}" +MODEL_DESCRIPTION="" if [[ ! -f "${MODEL_PATH}/config.json" ]]; then echo "ERROR: 模型目录不完整:${MODEL_PATH}/config.json 不存在" >&2 @@ -33,6 +28,30 @@ if [[ ! -f "${MODEL_PATH}/config.json" ]]; then exit 1 fi +if [[ "${AUTO_DETECT}" == "1" ]]; then + DETECT_OUTPUT="$(python3 "${DETECT_SCRIPT}" "${MODEL_PATH}")" || exit 1 + eval "${DETECT_OUTPUT}" + MODEL_KIND="${MODEL_KIND:-${DETECTED_MODEL_KIND}}" + TP="${TP:-${DETECTED_TP}}" + PP="${PP:-${DETECTED_PP}}" + PORT="${PORT:-${DETECTED_PORT}}" + SERVED_NAME="${SERVED_NAME:-${DETECTED_SERVED_NAME}}" + MODEL_DESCRIPTION="${DETECTED_DESCRIPTION:-}" +else + MODEL_KIND="${MODEL_KIND:-unknown}" +fi + +HOST="${HOST:-0.0.0.0}" +TP="${TP:-8}" +PP="${PP:-1}" +PORT="${PORT:-8091}" +SERVED_NAME="${SERVED_NAME:-llama4}" +MAX_MODEL_LEN="${MAX_MODEL_LEN:-4096}" +MAX_NUM_SEQS="${MAX_NUM_SEQS:-1}" +GPU_MEM_UTIL="${GPU_MEM_UTIL:-0.90}" +ENFORCE_EAGER="${ENFORCE_EAGER:-1}" +EXTRA_ARGS="${EXTRA_ARGS:-}" + PP_ARGS=() if (( PP > 1 )); then PP_ARGS=(--pipeline-parallel-size "${PP}") @@ -45,6 +64,7 @@ fi echo "============================================================" echo "Llama 4 (BI-V150 unified image)" +echo " model kind : ${MODEL_KIND}" echo " model : ${MODEL_PATH}" echo " served name: ${SERVED_NAME}" echo " listen : ${HOST}:${PORT}" @@ -55,6 +75,9 @@ echo " gpu util : ${GPU_MEM_UTIL}" echo " eager : ${ENFORCE_EAGER}" echo " visible : ${CUDA_VISIBLE_DEVICES:-all}" echo " extra : ${EXTRA_ARGS}" +if [[ -n "${MODEL_DESCRIPTION}" ]]; then + echo " detected : ${MODEL_DESCRIPTION}" +fi echo "============================================================" # shellcheck disable=SC2086 diff --git a/presets/maverick.env b/presets/maverick.env deleted file mode 100644 index 00b3e95..0000000 --- a/presets/maverick.env +++ /dev/null @@ -1,12 +0,0 @@ -# Llama 4 Maverick FP8 默认配置 -DEFAULT_IMAGE="llama4-bi150:2026-10-26" -DEFAULT_NAME="llama4-maverick" -DEFAULT_PORT="8092" -DEFAULT_MODEL_HOST_DIR="/mnt/disk0/models/LLM-Research/Llama-4-Maverick-17B-128E-Instruct-FP8" -DEFAULT_SERVED_NAME="llama4-maverick" -DEFAULT_TP="8" -DEFAULT_PP="2" -DEFAULT_GPUS="0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15" -DEFAULT_MAX_MODEL_LEN="4096" -DEFAULT_MAX_NUM_SEQS="1" -DEFAULT_GPU_MEM_UTIL="0.90" diff --git a/presets/scout.env b/presets/scout.env deleted file mode 100644 index 6fae6af..0000000 --- a/presets/scout.env +++ /dev/null @@ -1,12 +0,0 @@ -# Llama 4 Scout 默认配置 -DEFAULT_IMAGE="llama4-bi150:2026-10-26" -DEFAULT_NAME="llama4-scout" -DEFAULT_PORT="8091" -DEFAULT_MODEL_HOST_DIR="/mnt/disk0/models/LLM-Research/Llama-4-Scout-17B-16E-Instruct" -DEFAULT_SERVED_NAME="llama4-scout" -DEFAULT_TP="8" -DEFAULT_PP="1" -DEFAULT_GPUS="0,1,2,3,4,5,6,7" -DEFAULT_MAX_MODEL_LEN="4096" -DEFAULT_MAX_NUM_SEQS="1" -DEFAULT_GPU_MEM_UTIL="0.90" diff --git a/run.sh b/run.sh index 737f6e6..a920623 100755 --- a/run.sh +++ b/run.sh @@ -1,17 +1,24 @@ #!/usr/bin/env bash # -# Llama 4 Scout / Maverick 一键构建 + 启动(Iluvatar BI-V150) +# Llama 4 统一构建 + 启动(Iluvatar BI-V150) # -# ./run.sh scout -# ./run.sh maverick +# 自动读取模型目录下的 config.json,判断是 Scout 还是 Maverick,然后选择: +# Scout : TP8 / PP1 / 8 卡 / 8091 / llama4-scout +# Maverick : TP8 / PP2 / 16 卡 / 8092 / llama4-maverick +# +# 用法: +# ./run.sh /mnt/disk0/models/LLM-Research/Llama-4-Scout-17B-16E-Instruct +# ./run.sh /mnt/disk0/models/LLM-Research/Llama-4-Maverick-17B-128E-Instruct-FP8 +# MODEL_HOST_DIR=/path/to/model ./run.sh +# ./run.sh # 依次尝试两个默认模型目录 # # 常用覆盖项: -# GPUS=8,9,10,11,12,13,14,15 ./run.sh scout -# MODEL_HOST_DIR=/path/to/model ./run.sh maverick -# EXTRA_ARGS="--max-model-len 8192" ./run.sh scout -# EXTRA_DOCKER_ARGS="-v /mnt/disk1:/data:ro" ./run.sh scout -# BUILD=0 ./run.sh scout # 使用已有镜像,不重新构建 -# DRY_RUN=1 ./run.sh scout # 只打印 docker run 命令 +# GPUS=8,9,10,11,12,13,14,15 ./run.sh /path/to/scout +# EXTRA_DOCKER_ARGS="-v /mnt/disk1:/data:ro" ./run.sh /path/to/model +# EXTRA_ARGS="--max-model-len 8192" ./run.sh /path/to/model +# TP=8 PP=1 PORT=8095 SERVED_NAME=my-llama4 ./run.sh /path/to/model +# BUILD=0 ./run.sh /path/to/model # 使用已有镜像 +# DRY_RUN=1 ./run.sh /path/to/model # 只打印 docker run 命令 # set -euo pipefail @@ -19,67 +26,91 @@ cd "$(dirname "$0")" usage() { cat >&2 <<'EOF' -用法: ./run.sh +用法: ./run.sh [MODEL_DIR] -预设默认值在 presets/.env 中,所有值都可以用环境变量覆盖: - IMAGE NAME PORT MODEL_HOST_DIR MODEL_MOUNT SERVED_NAME +不传 MODEL_DIR 时,按顺序尝试: + /mnt/disk0/models/LLM-Research/Llama-4-Scout-17B-16E-Instruct + /mnt/disk0/models/LLM-Research/Llama-4-Maverick-17B-128E-Instruct-FP8 + +模型类型由 MODEL_DIR/config.json 自动判断,不需要 scout/maverick 参数。 + +环境变量覆盖: + IMAGE NAME MODEL_HOST_DIR MODEL_MOUNT PORT SERVED_NAME TP PP GPUS MAX_MODEL_LEN MAX_NUM_SEQS GPU_MEM_UTIL - ENFORCE_EAGER EXTRA_ARGS EXTRA_DOCKER_ARGS + ENFORCE_EAGER AUTO_DETECT EXTRA_ARGS EXTRA_DOCKER_ARGS BUILD=0/1 DRY_RUN=0/1 EOF } -VARIANT="${1:-}" -if [[ -z "${VARIANT}" || "${VARIANT}" == "-h" || "${VARIANT}" == "--help" ]]; then +if [[ "${1:-}" == "-h" || "${1:-}" == "--help" ]]; then + usage + exit 0 +fi + +MODEL_HOST_DIR="${1:-${MODEL_HOST_DIR:-}}" +if [[ -z "${MODEL_HOST_DIR}" ]]; then + for candidate in \ + "/mnt/disk0/models/LLM-Research/Llama-4-Scout-17B-16E-Instruct" \ + "/mnt/disk0/models/LLM-Research/Llama-4-Maverick-17B-128E-Instruct-FP8"; do + if [[ -f "${candidate}/config.json" ]]; then + MODEL_HOST_DIR="${candidate}" + break + fi + done +fi + +if [[ -z "${MODEL_HOST_DIR}" ]]; then + echo "ERROR: 未指定模型目录,也没有找到默认模型目录。" >&2 usage exit 2 fi -PRESET="presets/${VARIANT}.env" -if [[ ! -f "${PRESET}" ]]; then - echo "ERROR: 未知预设: ${VARIANT}" >&2 - usage - exit 2 +if [[ ! -f "${MODEL_HOST_DIR}/config.json" ]]; then + echo "ERROR: 模型目录不完整: ${MODEL_HOST_DIR}/config.json" >&2 + echo "请先下载模型,或把 MODEL_DIR 指向正确目录。" >&2 + exit 1 fi -# shellcheck disable=SC1090 -source "${PRESET}" +DETECT_OUTPUT="$(python3 detect_model.py "${MODEL_HOST_DIR}")" || exit 1 +eval "${DETECT_OUTPUT}" -IMAGE="${IMAGE:-${DEFAULT_IMAGE}}" -NAME="${NAME:-${DEFAULT_NAME}}" -PORT="${PORT:-${DEFAULT_PORT}}" -MODEL_HOST_DIR="${MODEL_HOST_DIR:-${DEFAULT_MODEL_HOST_DIR}}" +MODEL_KIND="${DETECTED_MODEL_KIND}" +IMAGE="${IMAGE:-llama4-bi150:2026-10-26}" +NAME="${NAME:-llama4-${MODEL_KIND}}" +PORT="${PORT:-${DETECTED_PORT}}" MODEL_MOUNT="${MODEL_MOUNT:-/models/llama4}" -SERVED_NAME="${SERVED_NAME:-${DEFAULT_SERVED_NAME}}" -TP="${TP:-${DEFAULT_TP}}" -PP="${PP:-${DEFAULT_PP}}" -GPUS="${GPUS:-${CUDA_VISIBLE_DEVICES:-${DEFAULT_GPUS}}}" -MAX_MODEL_LEN="${MAX_MODEL_LEN:-${DEFAULT_MAX_MODEL_LEN}}" -MAX_NUM_SEQS="${MAX_NUM_SEQS:-${DEFAULT_MAX_NUM_SEQS}}" -GPU_MEM_UTIL="${GPU_MEM_UTIL:-${DEFAULT_GPU_MEM_UTIL}}" +SERVED_NAME="${SERVED_NAME:-${DETECTED_SERVED_NAME}}" +TP="${TP:-${DETECTED_TP}}" +PP="${PP:-${DETECTED_PP}}" +MAX_MODEL_LEN="${MAX_MODEL_LEN:-4096}" +MAX_NUM_SEQS="${MAX_NUM_SEQS:-1}" +GPU_MEM_UTIL="${GPU_MEM_UTIL:-0.90}" ENFORCE_EAGER="${ENFORCE_EAGER:-1}" +AUTO_DETECT="${AUTO_DETECT:-1}" EXTRA_ARGS="${EXTRA_ARGS:-}" EXTRA_DOCKER_ARGS="${EXTRA_DOCKER_ARGS:-}" BUILD="${BUILD:-1}" DRY_RUN="${DRY_RUN:-0}" +GPU_COUNT=$((TP * PP)) + +if [[ -n "${GPUS:-}" ]]; then + GPUS="${GPUS}" +elif [[ -n "${CUDA_VISIBLE_DEVICES:-}" ]]; then + GPUS="${CUDA_VISIBLE_DEVICES}" +else + GPUS="$(seq -s, 0 $((GPU_COUNT - 1)))" +fi GPUS="${GPUS// /}" -if [[ ! -f "${MODEL_HOST_DIR}/config.json" ]]; then - echo "ERROR: 模型目录不完整: ${MODEL_HOST_DIR}/config.json" >&2 - echo "请先下载模型,或设置 MODEL_HOST_DIR 指向正确目录。" >&2 - exit 1 -fi - -EXPECTED_GPUS=$((TP * PP)) if [[ -n "${GPUS}" ]]; then IFS=',' read -r -a GPU_LIST <<< "${GPUS}" - if (( ${#GPU_LIST[@]} < EXPECTED_GPUS )); then - echo "ERROR: TP*PP=${EXPECTED_GPUS},但 GPUS 只给了 ${#GPU_LIST[@]} 张: ${GPUS}" >&2 + if (( ${#GPU_LIST[@]} < GPU_COUNT )); then + echo "ERROR: ${MODEL_KIND} 需要 TP*PP=${GPU_COUNT} 张卡,但 GPUS 只给了 ${#GPU_LIST[@]} 张: ${GPUS}" >&2 exit 1 fi - if (( ${#GPU_LIST[@]} > EXPECTED_GPUS )); then - echo "WARN: TP*PP=${EXPECTED_GPUS},但 GPUS 给了 ${#GPU_LIST[@]} 张;vLLM 只会使用其中 ${EXPECTED_GPUS} 张。" >&2 + if (( ${#GPU_LIST[@]} > GPU_COUNT )); then + echo "WARN: TP*PP=${GPU_COUNT},但 GPUS 给了 ${#GPU_LIST[@]} 张;vLLM 只会使用其中 ${GPU_COUNT} 张。" >&2 fi fi @@ -101,6 +132,8 @@ DOCKER_RUN=( --ulimit stack=67108864 -e CUDA_VISIBLE_DEVICES="${GPUS}" -e MODEL_PATH="${MODEL_MOUNT}" + -e AUTO_DETECT="${AUTO_DETECT}" + -e MODEL_KIND="${MODEL_KIND}" -e SERVED_NAME="${SERVED_NAME}" -e PORT="${PORT}" -e TP="${TP}" @@ -115,6 +148,11 @@ DOCKER_RUN=( "${IMAGE}" ) +echo "==> detected ${MODEL_KIND}: ${DETECTED_DESCRIPTION}" +echo " model : ${MODEL_HOST_DIR}" +echo " TP/PP : ${TP}/${PP} (${GPU_COUNT} GPUs), port=${PORT}, name=${SERVED_NAME}" +echo " GPUs : ${GPUS}" + if [[ "${DRY_RUN}" == "1" ]]; then printf '%q ' "${DOCKER_RUN[@]}" printf '\n' @@ -131,7 +169,7 @@ fi echo "==> recreate ${NAME}" docker rm -f "${NAME}" 2>/dev/null || true -echo "==> run ${VARIANT}: GPUs=${GPUS}, TP=${TP}, PP=${PP}, model=${MODEL_HOST_DIR}" +echo "==> run ${MODEL_KIND}" "${DOCKER_RUN[@]}" echo "==> wait for http://127.0.0.1:${PORT}/health" diff --git a/verify.sh b/verify.sh new file mode 100755 index 0000000..6886400 --- /dev/null +++ b/verify.sh @@ -0,0 +1,355 @@ +#!/usr/bin/env bash +# +# Llama 4 Scout / Maverick 服务验证脚本 +# +# 默认行为: +# 1. 如果 docker 可用,自动找正在运行的 llama4-bi150 容器; +# 2. 从容器挂载/环境变量或 MODEL_HOST_DIR 推断模型目录、端口、served name; +# 3. 如果模型目录可读,调用 detect_model.py 自动判断 Scout / Maverick; +# 4. 依次检查 /health、/v1/models、/v1/chat/completions; +# 5. 可选 --with-reasoning 和 --with-image。 +# +# 常用: +# ./verify.sh +# ./verify.sh --model-dir /mnt/disk0/models/LLM-Research/Llama-4-Maverick-17B-128E-Instruct-FP8 --with-image +# ./verify.sh --container llama4-scout --with-reasoning +# ./verify.sh --base-url http://127.0.0.1:8092 --served-name llama4-maverick +# +set -euo pipefail + +cd "$(dirname "$0")" + +MODEL_DIR="${MODEL_HOST_DIR:-}" +CONTAINER="${CONTAINER:-}" +PORT="${PORT:-}" +SERVED_NAME="${SERVED_NAME:-}" +BASE_URL="${BASE_URL:-}" +TIMEOUT="${TIMEOUT:-180}" +WITH_IMAGE=0 +WITH_REASONING=0 +SKIP_CHAT=0 +KIND="unknown" + +usage() { + cat >&2 <<'EOF' +用法: ./verify.sh [options] + + --model-dir PATH 模型目录(含 config.json),用于自动判断 Scout/Maverick + --container NAME 运行中的容器名;默认自动查找 llama4-bi150 容器 + --port PORT 服务端口;默认从模型/容器推断 + --served-name NAME OpenAI API 模型名;默认从模型/容器推断 + --base-url URL 直接指定服务地址,例如 http://127.0.0.1:8092 + --timeout SEC 等待和请求超时,默认 180 + --with-reasoning 额外跑一个简单推理题(期望 27) + --with-image 额外跑图片识别(仅 Maverick 有意义) + --skip-chat 只检查 /health 和 /v1/models + -h, --help 显示帮助 + +环境变量也可以设置:MODEL_HOST_DIR CONTAINER PORT SERVED_NAME BASE_URL TIMEOUT +EOF +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --model-dir) MODEL_DIR="${2:?--model-dir 需要路径}"; shift 2 ;; + --container) CONTAINER="${2:?--container 需要名称}"; shift 2 ;; + --port) PORT="${2:?--port 需要端口}"; shift 2 ;; + --served-name) SERVED_NAME="${2:?--served-name 需要名称}"; shift 2 ;; + --base-url) BASE_URL="${2:?--base-url 需要 URL}"; shift 2 ;; + --timeout) TIMEOUT="${2:?--timeout 需要秒数}"; shift 2 ;; + --with-image) WITH_IMAGE=1; shift ;; + --with-reasoning) WITH_REASONING=1; shift ;; + --skip-chat) SKIP_CHAT=1; shift ;; + -h|--help) usage; exit 0 ;; + *) echo "ERROR: 未知参数: $1" >&2; usage; exit 2 ;; + esac +done + +fail() { echo "FAIL: $*" >&2; exit 1; } +pass() { echo "PASS: $*"; } + +command -v curl >/dev/null 2>&1 || fail "缺少 curl" +command -v python3 >/dev/null 2>&1 || fail "缺少 python3" + +TMP="$(mktemp -d)" +trap 'rm -rf "${TMP}"' EXIT + +# ---------------------------------------------------------------- container +if [[ -z "${CONTAINER}" ]] && command -v docker >/dev/null 2>&1; then + CONTAINER="$(docker ps --format '{{.Names}} {{.Image}}' 2>/dev/null \ + | awk '/llama4-bi150|llama4-scout|llama4-maverick/ {print $1; exit}' || true)" +fi + +container_env() { + local name="$1" key="$2" + docker inspect "${name}" --format '{{range .Config.Env}}{{println .}}{{end}}' 2>/dev/null \ + | awk -F= -v key="${key}" '$1 == key { sub(/^[^=]*=/, ""); print; exit }' || true +} + +if [[ -n "${CONTAINER}" ]] && command -v docker >/dev/null 2>&1; then + if [[ -z "${MODEL_DIR}" ]]; then + MODEL_DIR="$(docker inspect "${CONTAINER}" --format '{{json .Mounts}}' 2>/dev/null \ + | python3 -c ' +import json, sys +try: + mounts = json.load(sys.stdin) +except Exception: + sys.exit(0) +for m in mounts: + if m.get("Destination") == "/models/llama4": + print(m.get("Source", "")) + break +' || true)" + fi + [[ -z "${PORT}" ]] && PORT="$(container_env "${CONTAINER}" PORT)" + [[ -z "${SERVED_NAME}" ]] && SERVED_NAME="$(container_env "${CONTAINER}" SERVED_NAME)" +fi + +# ---------------------------------------------------------------- model dir +if [[ -z "${MODEL_DIR}" ]]; then + for candidate in \ + "/mnt/disk0/models/LLM-Research/Llama-4-Scout-17B-16E-Instruct" \ + "/mnt/disk0/models/LLM-Research/Llama-4-Maverick-17B-128E-Instruct-FP8"; do + if [[ -f "${candidate}/config.json" ]]; then + MODEL_DIR="${candidate}" + break + fi + done +fi + +DETECT_JSON="" +if [[ -n "${MODEL_DIR}" ]]; then + if [[ -f "${MODEL_DIR}/config.json" ]]; then + DETECT_JSON="$(python3 detect_model.py "${MODEL_DIR}" --json)" || { + echo "ERROR: 无法识别模型: ${MODEL_DIR}" >&2 + exit 1 + } + elif [[ -n "${CONTAINER}" ]] && command -v docker >/dev/null 2>&1; then + # 宿主机路径不可读时,退回到容器内检测 + DETECT_JSON="$(docker exec "${CONTAINER}" python3 /opt/llama4/detect_model.py /models/llama4 --json 2>/dev/null || true)" + else + fail "模型目录缺少 config.json: ${MODEL_DIR}" + fi +fi + +if [[ -n "${DETECT_JSON}" ]]; then + KIND="$(python3 -c 'import json,sys; print(json.load(sys.stdin)["kind"])' <<< "${DETECT_JSON}")" + DETECTED_PORT="$(python3 -c 'import json,sys; print(json.load(sys.stdin)["port"])' <<< "${DETECT_JSON}")" + DETECTED_NAME="$(python3 -c 'import json,sys; print(json.load(sys.stdin)["served_name"])' <<< "${DETECT_JSON}")" + [[ -z "${PORT}" ]] && PORT="${DETECTED_PORT}" + [[ -z "${SERVED_NAME}" ]] && SERVED_NAME="${DETECTED_NAME}" +fi + +PORT="${PORT:-8091}" +SERVED_NAME="${SERVED_NAME:-llama4}" +BASE_URL="${BASE_URL:-http://127.0.0.1:${PORT}}" +BASE_URL="${BASE_URL%/}" + +echo "=== Llama 4 verification ===" +echo "container : ${CONTAINER:-}" +echo "model dir : ${MODEL_DIR:-}" +echo "kind : ${KIND}" +echo "base URL : ${BASE_URL}" +echo "model name: ${SERVED_NAME}" +echo + +# ---------------------------------------------------------------- health +echo "==> waiting for /health (up to ${TIMEOUT}s)" +HEALTH_OK=0 +DEADLINE=$((SECONDS + TIMEOUT)) +while (( SECONDS < DEADLINE )); do + if curl -fsS --max-time 5 "${BASE_URL}/health" > "${TMP}/health.txt" 2>/dev/null; then + HEALTH_OK=1 + break + fi + sleep 2 +done +if [[ "${HEALTH_OK}" != "1" ]]; then + fail "health check 超时: ${BASE_URL}/health" +fi +HEALTH="$(cat "${TMP}/health.txt" 2>/dev/null || true)" +pass "health: ${HEALTH:-OK}" + +# ---------------------------------------------------------------- models +echo "==> /v1/models" +curl -fsS --max-time "${TIMEOUT}" "${BASE_URL}/v1/models" > "${TMP}/models.json" \ + || fail "GET /v1/models 失败" +python3 - "${TMP}/models.json" "${SERVED_NAME}" <<'PY' +import json +import sys + +data = json.load(open(sys.argv[1])) +expected = sys.argv[2] +ids = [m.get("id") for m in data.get("data", [])] +if expected not in ids: + print(f"FAIL: /v1/models 中没有 {expected!r}; 实际: {ids}", file=sys.stderr) + sys.exit(1) +for m in data.get("data", []): + if m.get("id") == expected: + print(f"PASS: model={expected}, max_model_len={m.get('max_model_len')}") + break +PY + +if [[ "${SKIP_CHAT}" == "1" ]]; then + echo + echo "All checks passed (chat skipped)." + exit 0 +fi + +# ---------------------------------------------------------------- chat +echo "==> /v1/chat/completions" +python3 - "${SERVED_NAME}" > "${TMP}/chat.json" <<'PY' +import json +import sys + +print(json.dumps({ + "model": sys.argv[1], + "messages": [{"role": "user", "content": "你好,请用一句中文介绍你自己。"}], + "max_tokens": 64, + "temperature": 0, +}, ensure_ascii=False)) +PY +curl -fsS --max-time "${TIMEOUT}" \ + -H 'Content-Type: application/json' \ + --data-binary @"${TMP}/chat.json" \ + "${BASE_URL}/v1/chat/completions" > "${TMP}/chat_resp.json" \ + || fail "chat completion 失败" +python3 - "${TMP}/chat_resp.json" <<'PY' +import json +import sys + +data = json.load(open(sys.argv[1])) +try: + content = data["choices"][0]["message"]["content"] +except (KeyError, IndexError) as exc: + print(f"FAIL: unexpected chat response: {data}", file=sys.stderr) + sys.exit(1) +if isinstance(content, list): + content = " ".join( + part.get("text", "") for part in content if isinstance(part, dict) + ) +content = (content or "").strip() +if not content: + print("FAIL: chat response content is empty", file=sys.stderr) + sys.exit(1) +print(f"PASS: chat: {content[:200]}") +PY + +# ---------------------------------------------------------------- reasoning +if [[ "${WITH_REASONING}" == "1" ]]; then + echo "==> reasoning" + python3 - "${SERVED_NAME}" > "${TMP}/reason.json" <<'PY' +import json +import sys + +print(json.dumps({ + "model": sys.argv[1], + "messages": [{"role": "user", "content": "3 个人 3 天喝 3 桶水,9 个人 9 天喝几桶水?只给最终数字。"}], + "max_tokens": 16, + "temperature": 0, +}, ensure_ascii=False)) +PY + curl -fsS --max-time "${TIMEOUT}" \ + -H 'Content-Type: application/json' \ + --data-binary @"${TMP}/reason.json" \ + "${BASE_URL}/v1/chat/completions" > "${TMP}/reason_resp.json" \ + || fail "reasoning 请求失败" + python3 - "${TMP}/reason_resp.json" <<'PY' +import json +import sys + +data = json.load(open(sys.argv[1])) +content = data["choices"][0]["message"].get("content", "") +if isinstance(content, list): + content = " ".join( + part.get("text", "") for part in content if isinstance(part, dict) + ) +content = (content or "").strip() +if "27" not in content: + print(f"FAIL: expected 27, got: {content!r}", file=sys.stderr) + sys.exit(1) +print(f"PASS: reasoning: {content[:100]}") +PY +fi + +# ---------------------------------------------------------------- image +if [[ "${WITH_IMAGE}" == "1" ]]; then + if [[ "${KIND}" != "maverick" && "${KIND}" != "unknown" ]]; then + echo "SKIP: image test 仅适用于 Maverick(当前识别为 ${KIND})" + else + echo "==> image (336x336 red PNG)" + python3 - "${TMP}/red.b64" <<'PY' +import base64 +import struct +import sys +import zlib + +w = h = 336 +row = bytes((220, 20, 20)) * w +raw = b"".join(b"\x00" + row for _ in range(h)) + +def chunk(kind, data): + return ( + struct.pack(">I", len(data)) + + kind + + data + + struct.pack(">I", zlib.crc32(kind + data) & 0xFFFFFFFF) + ) + +png = ( + b"\x89PNG\r\n\x1a\n" + + chunk(b"IHDR", struct.pack(">IIBBBBB", w, h, 8, 2, 0, 0, 0)) + + chunk(b"IDAT", zlib.compress(raw, 9)) + + chunk(b"IEND", b"") +) +open(sys.argv[1], "w").write(base64.b64encode(png).decode()) +PY + python3 - "${SERVED_NAME}" "${TMP}/red.b64" > "${TMP}/image.json" <<'PY' +import json +import sys + +b64 = open(sys.argv[2]).read().strip() +payload = { + "model": sys.argv[1], + "messages": [{ + "role": "user", + "content": [ + {"type": "text", "text": "这张图片主要是什么颜色?只回答颜色。"}, + {"type": "image_url", "image_url": {"url": "data:image/png;base64," + b64}}, + ], + }], + "max_tokens": 32, + "temperature": 0, +} +print(json.dumps(payload, ensure_ascii=False)) +PY + curl -fsS --max-time "${TIMEOUT}" \ + -H 'Content-Type: application/json' \ + --data-binary @"${TMP}/image.json" \ + "${BASE_URL}/v1/chat/completions" > "${TMP}/image_resp.json" \ + || fail "image 请求失败" + python3 - "${TMP}/image_resp.json" <<'PY' +import json +import sys + +data = json.load(open(sys.argv[1])) +content = data["choices"][0]["message"].get("content", "") +if isinstance(content, list): + content = " ".join( + part.get("text", "") for part in content if isinstance(part, dict) + ) +content = (content or "").strip() +if not content: + print("FAIL: image response content is empty", file=sys.stderr) + sys.exit(1) +lower = content.lower() +if "红" not in lower and "red" not in lower: + print(f"FAIL: expected red, got: {content[:200]}", file=sys.stderr) + sys.exit(1) +print(f"PASS: image: {content[:200]}") +PY + fi +fi + +echo +echo "All checks passed for ${KIND} at ${BASE_URL}."