diff --git a/.gitea/workflows/docker-build-push.yml b/.gitea/workflows/docker-build-push.yml index 48bd35d..dfa60e8 100644 --- a/.gitea/workflows/docker-build-push.yml +++ b/.gitea/workflows/docker-build-push.yml @@ -15,7 +15,9 @@ jobs: git checkout "${{ gitea.ref_name }}" - name: Run compatibility tests - run: python3 -m unittest discover -s tests -q + run: | + python3 -m unittest discover -s tests -q + bash -n vllm_wrapper.sh - name: Load image metadata run: | @@ -60,6 +62,8 @@ jobs: image_base="$DOCKER_REGISTRY/$DOCKER_USERNAME/$image_name" a100_image="$image_base:${{ gitea.ref_name }}-nvidia-a100" cambricon_image="$image_base:${{ gitea.ref_name }}-cambricon-mlu370" + metax_image="$image_base:${{ gitea.ref_name }}-metax-c500" + vastai_image="$image_base:${{ gitea.ref_name }}-vastai-va16" docker build \ --build-arg BASE_IMAGE=harbor.4pd.io/dooke/vllm/vllm/vllm-openai:v0.11.0 \ @@ -67,8 +71,14 @@ jobs: docker build \ --build-arg BASE_IMAGE=harbor.4pd.io/hardcore-tech/cambricon-mlu370-pytorch:v25.01-20260228 \ -t "$cambricon_image" . + docker build \ + --build-arg BASE_IMAGE=harbor.4pd.io/modelhubxc/enginex-metax/vllm:0.9.1 \ + -t "$metax_image" . + docker build \ + --build-arg BASE_IMAGE=harbor.4pd.io/modelhubxc/sunruoxi/enginex-vastai-vllm-vacc-fix-tokenizer:VVI-26.02 \ + -t "$vastai_image" . - for image in "$a100_image" "$cambricon_image"; do + for image in "$a100_image" "$cambricon_image" "$metax_image" "$vastai_image"; do success=0 for attempt in 1 2 3; do if docker push "$image"; then success=1; break; fi @@ -79,6 +89,8 @@ jobs: done echo "A100_IMAGE=$a100_image" >> "$GITEA_ENV" echo "CAMBRICON_IMAGE=$cambricon_image" >> "$GITEA_ENV" + echo "METAX_IMAGE=$metax_image" >> "$GITEA_ENV" + echo "VASTAI_IMAGE=$vastai_image" >> "$GITEA_ENV" - name: Notify ModelHub image verification run: | @@ -87,12 +99,16 @@ jobs: case "$gpu" in Nvidia-A100) image="$A100_IMAGE" ;; Cambricon_mlu-370-x4|Cambricon_mlu-370-x8) image="$CAMBRICON_IMAGE" ;; + MetaX_c-500) image="$METAX_IMAGE" ;; + Vastai_va16) image="$VASTAI_IMAGE" ;; *) echo "unknown GPU target: $gpu"; exit 1 ;; esac payload="$(GPU_TYPE="$gpu" IMAGE="$image" REPO_URL='${{ gitea.server_url }}/${{ gitea.repository }}' TAG='${{ gitea.ref_name }}' ACTOR='${{ gitea.actor }}' python3 -c 'import json, os; print(json.dumps({"framework": os.environ["FRAMEWORK"], "gpuType": os.environ["GPU_TYPE"], "imageUrl": os.environ["IMAGE"], "taskType": os.environ["TASK_TYPE"], "createBy": os.environ["ACTOR"], "repoUrl": os.environ["REPO_URL"], "tag": os.environ["TAG"]}))')" - curl --silent --show-error --fail-with-body --location \ + response="$(curl --silent --show-error --fail-with-body --location \ --request POST 'https://modelhub.org.cn/adminApi/image-verify' \ --header "Xc-Token: $FIXED_TOKEN" \ --header 'Content-Type: application/json' \ - --data-raw "$payload" + --data-raw "$payload")" + VERIFY_RESPONSE="$response" python3 -c 'import json,os,sys; body=json.loads(os.environ["VERIFY_RESPONSE"]); sys.exit(0 if body.get("code")==0 else 1)' + echo "submitted image verification for $gpu" done diff --git a/.gitea/workflows/task_info.env b/.gitea/workflows/task_info.env index e6f7f2f..66a6498 100644 --- a/.gitea/workflows/task_info.env +++ b/.gitea/workflows/task_info.env @@ -1,3 +1,3 @@ FRAMEWORK=vllm_qwen25_compat TASK_TYPE=text-generation -GPU_TYPES="Nvidia-A100 Cambricon_mlu-370-x4 Cambricon_mlu-370-x8" +GPU_TYPES="Nvidia-A100 Cambricon_mlu-370-x4 Cambricon_mlu-370-x8 MetaX_c-500 Vastai_va16" diff --git a/README.md b/README.md index 45637c1..7fa6521 100644 --- a/README.md +++ b/README.md @@ -4,10 +4,10 @@ 的方式:标签触发 `.gitea/workflows/docker-build-push.yml`,runner 检查元数据、构建并推送镜像, 最后调用 ModelHub 的 `image-verify` 接口注册验证。 -当前为 A100 参考端和寒武纪 MLU370 x4/x8 构建两个镜像,三个 GPU 共用 -`vllm_qwen25_compat` 框架名。A100 基础镜像来自示例仓库;寒武纪基础镜像来自 -平台现有 MLU370 验证任务使用的镜像。是否允许一个仓库为多个 GPU 注册同一框架, -需要以 runner 的 `validate` 和镜像验证结果为准。 +当前为 A100 参考端、寒武纪 MLU370 x4/x8、沐曦 C500 和 Vastai VA16 +构建四个镜像,五个 GPU 共用 `vllm_qwen25_compat` 框架名。基础镜像均来自 +平台已有验证任务使用的版本;A100 沿用示例仓库的基础镜像。每个 GPU 的注册 +与可用性仍以平台镜像验证结果和 `frameworks` 接口回读为准。 ## 修复范围 @@ -19,9 +19,11 @@ 简单的 Qwen ChatML 模板,避免 `/v1/chat/completions` 返回 400;已有独立 `chat_template.jinja` 或 `chat_template.json` 的模型不补写。 -启动时在 `/tmp` 建立新模型视图,只重写 `tokenizer_config.json`。权重、模型代码和 -其他文件以符号链接指回只读 `/model`;源模型不变。没有命中上述问题时直接调用原始 -`vllm`。修复记录写在新视图的 `.modelhub-compat-report.json`,同时输出到容器日志。 +启动时在 `/tmp` 建立新视图,只重写 `tokenizer_config.json`。若基础镜像使用 +`--tokenizer` 指向单独的临时目录(Vastai 的现有启动脚本如此),则修复实际加载的 +tokenizer 目录并保留模型仓库的独立聊天模板,权重路径仍指向原模型;否则修复模型 +视图。源模型不变。没有命中问题时直接调用原始 `vllm`。修复记录写在新视图的 +`.modelhub-compat-report.json`,同时输出到容器日志。 这不能修复缺失权重、尺寸不匹配、Ascend 上持续出现的 0 分输出或平台评测超时。 尤其是分数为 0 的参考端与目标端必须分别排查;只换一个端的镜像不能保证过审。 @@ -40,16 +42,21 @@ docker build --build-arg BASE_IMAGE=harbor.4pd.io/dooke/vllm/vllm/vllm-openai:v0 -t qwen25-compat:a100 . docker build --build-arg BASE_IMAGE=harbor.4pd.io/hardcore-tech/cambricon-mlu370-pytorch:v25.01-20260228 \ -t qwen25-compat:mlu370 . +docker build --build-arg BASE_IMAGE=harbor.4pd.io/modelhubxc/enginex-metax/vllm:0.9.1 \ + -t qwen25-compat:metax . +docker build --build-arg BASE_IMAGE=harbor.4pd.io/modelhubxc/sunruoxi/enginex-vastai-vllm-vacc-fix-tokenizer:VVI-26.02 \ + -t qwen25-compat:vastai . ``` ## 用 runner 发布 -1. 源码已推送到公开仓库 `a4913994/enginex-qwen25-compat` 的 `main`;目前尚未推送触发构建的 `v*` 标签。 -2. 为该仓库的 runner 配置 `DOCKER_REGISTRY`、`DOCKER_USERNAME`、 - `DOCKER_PASSWORD`、`FIXED_TOKEN`。不要把凭证写进仓库。 -3. 推送 `v*` 标签。工作流依次跑单元测试、验证 GPU/任务元数据、构建与推送镜像、 - 为 A100 和 MLU370 x4/x8 各发起一次镜像验证。 -4. 在“我的驱动列表”确认三个镜像验证通过,并检查新框架出现在对应 GPU 的 +1. 源码在公开仓库 `a4913994/enginex-qwen25-compat` 的 `main`;`v0.1.0` + 已完成 A100/寒武纪镜像构建并提交镜像验证,验证结果须单独回读。 +2. runner 使用 `DOCKER_REGISTRY`、`DOCKER_USERNAME`、`DOCKER_PASSWORD`、 + `FIXED_TOKEN`;不要把凭证写进仓库。 +3. 推送新的 `v*` 标签后,工作流依次跑单元测试、验证 GPU/任务元数据、构建与推送 + 镜像,并为五个 GPU 各发起一次镜像验证;接口业务码也会被检查。 +4. 在“我的驱动列表”确认五个镜像验证通过,并检查新框架出现在对应 GPU 的 `frameworks` 接口中,再用少量已知缺陷模型做对照验证。 只有平台确认新框架可选且 A100 参考端实际使用新镜像后,才应让 `xc-agent` 把 diff --git a/engine_compat.py b/engine_compat.py index d8a81a8..5339d59 100644 --- a/engine_compat.py +++ b/engine_compat.py @@ -43,10 +43,11 @@ def _key(value: Any) -> str: def repair_tokenizer_config( - model_dir: Path, tokenizer: dict[str, Any] + model_dir: Path, tokenizer: dict[str, Any], *, tokenizer_dir: Path | None = None ) -> tuple[dict[str, Any], list[str]]: patched = dict(tokenizer) repairs: list[str] = [] + effective_tokenizer_dir = tokenizer_dir or model_dir extra = patched.get("extra_special_tokens") if isinstance(extra, list): @@ -76,7 +77,7 @@ def repair_tokenizer_config( # Only replace the known class mismatch from existing ModelHub failures. # Arbitrary custom tokenizer classes may require their own Python code. if patched.get("tokenizer_class") == "TokenizersBackend": - if not (model_dir / "tokenizer.json").is_file(): + if not (effective_tokenizer_dir / "tokenizer.json").is_file(): raise CompatibilityError("TokenizersBackend requires tokenizer.json") patched["tokenizer_class"] = "PreTrainedTokenizerFast" repairs.append("TokenizersBackend -> PreTrainedTokenizerFast") @@ -85,7 +86,8 @@ def repair_tokenizer_config( # tokenizer_config.json. Do not synthesize one when the repository already # supplies a template in that format. has_standalone_template = any( - (model_dir / name).is_file() + (directory / name).is_file() + for directory in (effective_tokenizer_dir, model_dir) for name in ("chat_template.jinja", "chat_template.json") ) if not patched.get("chat_template") and not has_standalone_template: @@ -102,15 +104,37 @@ def repair_tokenizer_config( return patched, repairs -def prepare_model(model_dir: str | Path, *, work_root: str | Path | None = None) -> str | None: - source = Path(model_dir).resolve() +def prepare_model( + model_dir: str | Path, + *, + tokenizer_dir: str | Path | None = None, + work_root: str | Path | None = None, +) -> str | None: + model_source = Path(model_dir).resolve() + if not model_source.is_dir(): + raise CompatibilityError(f"model directory is missing: {model_source}") + source = Path(tokenizer_dir).resolve() if tokenizer_dir is not None else model_source if not source.is_dir(): - raise CompatibilityError(f"model directory is missing: {source}") + raise CompatibilityError(f"tokenizer directory is missing: {source}") config_path = source / "tokenizer_config.json" if not config_path.is_file(): return None - patched, repairs = repair_tokenizer_config(source, _json_object(config_path)) + patched, repairs = repair_tokenizer_config( + model_source, _json_object(config_path), tokenizer_dir=source + ) + # Some vendor entrypoints copy tokenizer files to /tmp and pass that path + # explicitly with --tokenizer, omitting a standalone template in the copy. + # Keep the original template visible to the effective tokenizer directory. + extra_templates = [ + name + for name in ("chat_template.jinja", "chat_template.json") + if source != model_source + and not (source / name).is_file() + and (model_source / name).is_file() + ] + if extra_templates: + repairs.append("preserved standalone chat template") if not repairs: return None @@ -137,8 +161,17 @@ def prepare_model(model_dir: str | Path, *, work_root: str | Path | None = None) raise CompatibilityError( f"cannot link model item: {item.name}" ) from link_exc + for name in extra_templates: + (view / name).symlink_to(model_source / name) (view / ".modelhub-compat-report.json").write_text( - json.dumps({"source": str(source), "repairs": repairs}, ensure_ascii=False) + json.dumps( + { + "model_source": str(model_source), + "tokenizer_source": str(source), + "repairs": repairs, + }, + ensure_ascii=False, + ) + "\n", encoding="utf-8", ) @@ -153,11 +186,13 @@ def prepare_model(model_dir: str | Path, *, work_root: str | Path | None = None) def main() -> int: - if len(sys.argv) != 2: - print("usage: engine_compat.py MODEL_DIR", file=sys.stderr) + if len(sys.argv) not in (2, 3): + print("usage: engine_compat.py MODEL_DIR [TOKENIZER_DIR]", file=sys.stderr) return 2 try: - view = prepare_model(sys.argv[1]) + view = prepare_model( + sys.argv[1], tokenizer_dir=sys.argv[2] if len(sys.argv) == 3 else None + ) except CompatibilityError as exc: print(f"model compatibility error: {exc}", file=sys.stderr) return 2 diff --git a/tests/test_engine_compat.py b/tests/test_engine_compat.py index 62a1bc7..ca3a5a5 100644 --- a/tests/test_engine_compat.py +++ b/tests/test_engine_compat.py @@ -84,6 +84,47 @@ class EngineCompatTests(unittest.TestCase): ) self.assertIsNone(prepare_model(source, work_root=root)) + def test_explicit_tokenizer_directory_gets_qwen_template(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + model = self._model(root, {"eos_token": "<|im_end|>"}) + tokenizer = root / "vendor-tokenizer" + tokenizer.mkdir() + (tokenizer / "tokenizer_config.json").write_text( + json.dumps( + { + "eos_token": "<|im_end|>", + "extra_special_tokens": ["<|im_start|>"], + "tokenizer_class": "TokenizersBackend", + } + ), + encoding="utf-8", + ) + (tokenizer / "tokenizer.json").write_text("{}", encoding="utf-8") + view = Path(prepare_model(model, tokenizer_dir=tokenizer, work_root=root)) + fixed = json.loads((view / "tokenizer_config.json").read_text("utf-8")) + self.assertEqual(fixed["chat_template"], QWEN_CHAT_TEMPLATE) + self.assertEqual(fixed["additional_special_tokens"], ["<|im_start|>"]) + self.assertEqual(fixed["tokenizer_class"], "PreTrainedTokenizerFast") + self.assertEqual((model / "model.safetensors").read_bytes(), b"sample-weights") + self.assertFalse((view / "model.safetensors").exists()) + self.assertEqual((view / "tokenizer.json").read_text("utf-8"), "{}") + + def test_explicit_tokenizer_directory_preserves_model_standalone_template(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + model = self._model(root, {"eos_token": "<|im_end|>"}) + (model / "chat_template.jinja").write_text("{{ messages }}", encoding="utf-8") + tokenizer = root / "vendor-tokenizer" + tokenizer.mkdir() + (tokenizer / "tokenizer_config.json").write_text( + json.dumps({"eos_token": "<|im_end|>"}), encoding="utf-8" + ) + view = Path(prepare_model(model, tokenizer_dir=tokenizer, work_root=root)) + fixed = json.loads((view / "tokenizer_config.json").read_text("utf-8")) + self.assertNotIn("chat_template", fixed) + self.assertEqual((view / "chat_template.jinja").read_text("utf-8"), "{{ messages }}") + if __name__ == "__main__": unittest.main() diff --git a/vllm_wrapper.sh b/vllm_wrapper.sh index 96b71b3..86572f5 100644 --- a/vllm_wrapper.sh +++ b/vllm_wrapper.sh @@ -9,17 +9,39 @@ fi model_dir="$2" shift 2 -prepared="$(python3 /opt/modelhub/engine_compat.py "$model_dir")" +args=("$@") +tokenizer_dir="" +for ((i=0; i<${#args[@]}; i++)); do + if [[ "${args[i]}" == "--tokenizer" && $((i+1)) -lt ${#args[@]} ]]; then + tokenizer_dir="${args[i+1]}" + break + elif [[ "${args[i]}" == --tokenizer=* ]]; then + tokenizer_dir="${args[i]#--tokenizer=}" + break + fi +done + +if [[ -n "$tokenizer_dir" && -d "$tokenizer_dir" ]]; then + prepared="$(python3 /opt/modelhub/engine_compat.py "$model_dir" "$tokenizer_dir")" +else + prepared="$(python3 /opt/modelhub/engine_compat.py "$model_dir")" + tokenizer_dir="" +fi if [[ -z "$prepared" ]]; then - exec "$vllm_real" serve "$model_dir" "$@" + exec "$vllm_real" serve "$model_dir" "${args[@]}" fi -args=("$@") +effective_model="$prepared" +if [[ -n "$tokenizer_dir" && "$(readlink -f "$tokenizer_dir")" != "$(readlink -f "$model_dir")" ]]; then + # A vendor entrypoint's --tokenizer points at a copied tokenizer tree. + # Keep model weights at the original path; redirect only that tokenizer. + effective_model="$model_dir" +fi for ((i=0; i<${#args[@]}; i++)); do - if [[ "${args[i]}" == "--tokenizer" && $((i+1)) -lt ${#args[@]} && "${args[i+1]}" == "$model_dir" ]]; then + if [[ "${args[i]}" == "--tokenizer" && $((i+1)) -lt ${#args[@]} && "${args[i+1]}" == "$tokenizer_dir" ]]; then args[i+1]="$prepared" - elif [[ "${args[i]}" == "--tokenizer=$model_dir" ]]; then + elif [[ "${args[i]}" == "--tokenizer=$tokenizer_dir" && -n "$tokenizer_dir" ]]; then args[i]="--tokenizer=$prepared" fi done -exec "$vllm_real" serve "$prepared" "${args[@]}" +exec "$vllm_real" serve "$effective_model" "${args[@]}"