Support MetaX and Vastai tokenizer compatibility images

2026-09-30 12:16:08 +08:00
parent f8937cc505
commit fb5a163d07
6 changed files with 156 additions and 35 deletions
--- a/.gitea/workflows/docker-build-push.yml
+++ b/.gitea/workflows/docker-build-push.yml
@@ -15,7 +15,9 @@ jobs:
          git checkout "${{ gitea.ref_name }}"

      - name: Run compatibility tests
-        run: python3 -m unittest discover -s tests -q
+        run: |
+          python3 -m unittest discover -s tests -q
+          bash -n vllm_wrapper.sh

      - name: Load image metadata
        run: |
@@ -60,6 +62,8 @@ jobs:
          image_base="$DOCKER_REGISTRY/$DOCKER_USERNAME/$image_name"
          a100_image="$image_base:${{ gitea.ref_name }}-nvidia-a100"
          cambricon_image="$image_base:${{ gitea.ref_name }}-cambricon-mlu370"
+          metax_image="$image_base:${{ gitea.ref_name }}-metax-c500"
+          vastai_image="$image_base:${{ gitea.ref_name }}-vastai-va16"

          docker build \
            --build-arg BASE_IMAGE=harbor.4pd.io/dooke/vllm/vllm/vllm-openai:v0.11.0 \
@@ -67,8 +71,14 @@ jobs:
          docker build \
            --build-arg BASE_IMAGE=harbor.4pd.io/hardcore-tech/cambricon-mlu370-pytorch:v25.01-20260228 \
            -t "$cambricon_image" .
+          docker build \
+            --build-arg BASE_IMAGE=harbor.4pd.io/modelhubxc/enginex-metax/vllm:0.9.1 \
+            -t "$metax_image" .
+          docker build \
+            --build-arg BASE_IMAGE=harbor.4pd.io/modelhubxc/sunruoxi/enginex-vastai-vllm-vacc-fix-tokenizer:VVI-26.02 \
+            -t "$vastai_image" .

-          for image in "$a100_image" "$cambricon_image"; do
+          for image in "$a100_image" "$cambricon_image" "$metax_image" "$vastai_image"; do
            success=0
            for attempt in 1 2 3; do
              if docker push "$image"; then success=1; break; fi
@@ -79,6 +89,8 @@ jobs:
          done
          echo "A100_IMAGE=$a100_image" >> "$GITEA_ENV"
          echo "CAMBRICON_IMAGE=$cambricon_image" >> "$GITEA_ENV"
+          echo "METAX_IMAGE=$metax_image" >> "$GITEA_ENV"
+          echo "VASTAI_IMAGE=$vastai_image" >> "$GITEA_ENV"

      - name: Notify ModelHub image verification
        run: |
@@ -87,12 +99,16 @@ jobs:
            case "$gpu" in
              Nvidia-A100) image="$A100_IMAGE" ;;
              Cambricon_mlu-370-x4|Cambricon_mlu-370-x8) image="$CAMBRICON_IMAGE" ;;
+              MetaX_c-500) image="$METAX_IMAGE" ;;
+              Vastai_va16) image="$VASTAI_IMAGE" ;;
              *) echo "unknown GPU target: $gpu"; exit 1 ;;
            esac
            payload="$(GPU_TYPE="$gpu" IMAGE="$image" REPO_URL='${{ gitea.server_url }}/${{ gitea.repository }}' TAG='${{ gitea.ref_name }}' ACTOR='${{ gitea.actor }}' python3 -c 'import json, os; print(json.dumps({"framework": os.environ["FRAMEWORK"], "gpuType": os.environ["GPU_TYPE"], "imageUrl": os.environ["IMAGE"], "taskType": os.environ["TASK_TYPE"], "createBy": os.environ["ACTOR"], "repoUrl": os.environ["REPO_URL"], "tag": os.environ["TAG"]}))')"
-            curl --silent --show-error --fail-with-body --location \
+            response="$(curl --silent --show-error --fail-with-body --location \
              --request POST 'https://modelhub.org.cn/adminApi/image-verify' \
              --header "Xc-Token: $FIXED_TOKEN" \
              --header 'Content-Type: application/json' \
-              --data-raw "$payload"
+              --data-raw "$payload")"
+            VERIFY_RESPONSE="$response" python3 -c 'import json,os,sys; body=json.loads(os.environ["VERIFY_RESPONSE"]); sys.exit(0 if body.get("code")==0 else 1)'
+            echo "submitted image verification for $gpu"
          done
--- a/.gitea/workflows/task_info.env
+++ b/.gitea/workflows/task_info.env
@@ -1,3 +1,3 @@
 FRAMEWORK=vllm_qwen25_compat
 TASK_TYPE=text-generation
-GPU_TYPES="Nvidia-A100 Cambricon_mlu-370-x4 Cambricon_mlu-370-x8"
+GPU_TYPES="Nvidia-A100 Cambricon_mlu-370-x4 Cambricon_mlu-370-x8 MetaX_c-500 Vastai_va16"
--- a/README.md
+++ b/README.md
@@ -4,10 +4,10 @@
 的方式：标签触发 `.gitea/workflows/docker-build-push.yml`，runner 检查元数据、构建并推送镜像，
 最后调用 ModelHub 的 `image-verify` 接口注册验证。

-当前为 A100 参考端和寒武纪 MLU370 x4/x8 构建两个镜像，三个 GPU 共用
-`vllm_qwen25_compat` 框架名。A100 基础镜像来自示例仓库；寒武纪基础镜像来自
-平台现有 MLU370 验证任务使用的镜像。是否允许一个仓库为多个 GPU 注册同一框架，
-需要以 runner 的 `validate` 和镜像验证结果为准。
+当前为 A100 参考端、寒武纪 MLU370 x4/x8、沐曦 C500 和 Vastai VA16
+构建四个镜像，五个 GPU 共用 `vllm_qwen25_compat` 框架名。基础镜像均来自
+平台已有验证任务使用的版本；A100 沿用示例仓库的基础镜像。每个 GPU 的注册
+与可用性仍以平台镜像验证结果和 `frameworks` 接口回读为准。

 ## 修复范围

@@ -19,9 +19,11 @@
  简单的 Qwen ChatML 模板，避免 `/v1/chat/completions` 返回 400；已有独立
  `chat_template.jinja` 或 `chat_template.json` 的模型不补写。

-启动时在 `/tmp` 建立新模型视图，只重写 `tokenizer_config.json`。权重、模型代码和
-其他文件以符号链接指回只读 `/model`；源模型不变。没有命中上述问题时直接调用原始
-`vllm`。修复记录写在新视图的 `.modelhub-compat-report.json`，同时输出到容器日志。
+启动时在 `/tmp` 建立新视图，只重写 `tokenizer_config.json`。若基础镜像使用
+`--tokenizer` 指向单独的临时目录（Vastai 的现有启动脚本如此），则修复实际加载的
+tokenizer 目录并保留模型仓库的独立聊天模板，权重路径仍指向原模型；否则修复模型
+视图。源模型不变。没有命中问题时直接调用原始 `vllm`。修复记录写在新视图的
+`.modelhub-compat-report.json`，同时输出到容器日志。

 这不能修复缺失权重、尺寸不匹配、Ascend 上持续出现的 0 分输出或平台评测超时。
 尤其是分数为 0 的参考端与目标端必须分别排查；只换一个端的镜像不能保证过审。
@@ -40,16 +42,21 @@ docker build --build-arg BASE_IMAGE=harbor.4pd.io/dooke/vllm/vllm/vllm-openai:v0
  -t qwen25-compat:a100 .
 docker build --build-arg BASE_IMAGE=harbor.4pd.io/hardcore-tech/cambricon-mlu370-pytorch:v25.01-20260228 \
  -t qwen25-compat:mlu370 .
+docker build --build-arg BASE_IMAGE=harbor.4pd.io/modelhubxc/enginex-metax/vllm:0.9.1 \
+  -t qwen25-compat:metax .
+docker build --build-arg BASE_IMAGE=harbor.4pd.io/modelhubxc/sunruoxi/enginex-vastai-vllm-vacc-fix-tokenizer:VVI-26.02 \
+  -t qwen25-compat:vastai .
 ```

 ## 用 runner 发布

-1. 源码已推送到公开仓库 `a4913994/enginex-qwen25-compat` 的 `main`；目前尚未推送触发构建的 `v*` 标签。
-2. 为该仓库的 runner 配置 `DOCKER_REGISTRY`、`DOCKER_USERNAME`、
-   `DOCKER_PASSWORD`、`FIXED_TOKEN`。不要把凭证写进仓库。
-3. 推送 `v*` 标签。工作流依次跑单元测试、验证 GPU/任务元数据、构建与推送镜像、
-   为 A100 和 MLU370 x4/x8 各发起一次镜像验证。
-4. 在“我的驱动列表”确认三个镜像验证通过，并检查新框架出现在对应 GPU 的
+1. 源码在公开仓库 `a4913994/enginex-qwen25-compat` 的 `main`；`v0.1.0`
+   已完成 A100/寒武纪镜像构建并提交镜像验证，验证结果须单独回读。
+2. runner 使用 `DOCKER_REGISTRY`、`DOCKER_USERNAME`、`DOCKER_PASSWORD`、
+   `FIXED_TOKEN`；不要把凭证写进仓库。
+3. 推送新的 `v*` 标签后，工作流依次跑单元测试、验证 GPU/任务元数据、构建与推送
+   镜像，并为五个 GPU 各发起一次镜像验证；接口业务码也会被检查。
+4. 在“我的驱动列表”确认五个镜像验证通过，并检查新框架出现在对应 GPU 的
   `frameworks` 接口中，再用少量已知缺陷模型做对照验证。

 只有平台确认新框架可选且 A100 参考端实际使用新镜像后，才应让 `xc-agent` 把
--- a/engine_compat.py
+++ b/engine_compat.py
@@ -43,10 +43,11 @@ def _key(value: Any) -> str:


 def repair_tokenizer_config(
-    model_dir: Path, tokenizer: dict[str, Any]
+    model_dir: Path, tokenizer: dict[str, Any], *, tokenizer_dir: Path | None = None
 ) -> tuple[dict[str, Any], list[str]]:
    patched = dict(tokenizer)
    repairs: list[str] = []
+    effective_tokenizer_dir = tokenizer_dir or model_dir

    extra = patched.get("extra_special_tokens")
    if isinstance(extra, list):
@@ -76,7 +77,7 @@ def repair_tokenizer_config(
    # Only replace the known class mismatch from existing ModelHub failures.
    # Arbitrary custom tokenizer classes may require their own Python code.
    if patched.get("tokenizer_class") == "TokenizersBackend":
-        if not (model_dir / "tokenizer.json").is_file():
+        if not (effective_tokenizer_dir / "tokenizer.json").is_file():
            raise CompatibilityError("TokenizersBackend requires tokenizer.json")
        patched["tokenizer_class"] = "PreTrainedTokenizerFast"
        repairs.append("TokenizersBackend -> PreTrainedTokenizerFast")
@@ -85,7 +86,8 @@ def repair_tokenizer_config(
    # tokenizer_config.json. Do not synthesize one when the repository already
    # supplies a template in that format.
    has_standalone_template = any(
-        (model_dir / name).is_file()
+        (directory / name).is_file()
+        for directory in (effective_tokenizer_dir, model_dir)
        for name in ("chat_template.jinja", "chat_template.json")
    )
    if not patched.get("chat_template") and not has_standalone_template:
@@ -102,15 +104,37 @@ def repair_tokenizer_config(
    return patched, repairs


-def prepare_model(model_dir: str | Path, *, work_root: str | Path | None = None) -> str | None:
-    source = Path(model_dir).resolve()
+def prepare_model(
+    model_dir: str | Path,
+    *,
+    tokenizer_dir: str | Path | None = None,
+    work_root: str | Path | None = None,
+) -> str | None:
+    model_source = Path(model_dir).resolve()
+    if not model_source.is_dir():
+        raise CompatibilityError(f"model directory is missing: {model_source}")
+    source = Path(tokenizer_dir).resolve() if tokenizer_dir is not None else model_source
    if not source.is_dir():
-        raise CompatibilityError(f"model directory is missing: {source}")
+        raise CompatibilityError(f"tokenizer directory is missing: {source}")
    config_path = source / "tokenizer_config.json"
    if not config_path.is_file():
        return None

-    patched, repairs = repair_tokenizer_config(source, _json_object(config_path))
+    patched, repairs = repair_tokenizer_config(
+        model_source, _json_object(config_path), tokenizer_dir=source
+    )
+    # Some vendor entrypoints copy tokenizer files to /tmp and pass that path
+    # explicitly with --tokenizer, omitting a standalone template in the copy.
+    # Keep the original template visible to the effective tokenizer directory.
+    extra_templates = [
+        name
+        for name in ("chat_template.jinja", "chat_template.json")
+        if source != model_source
+        and not (source / name).is_file()
+        and (model_source / name).is_file()
+    ]
+    if extra_templates:
+        repairs.append("preserved standalone chat template")
    if not repairs:
        return None

@@ -137,8 +161,17 @@ def prepare_model(model_dir: str | Path, *, work_root: str | Path | None = None)
                        raise CompatibilityError(
                            f"cannot link model item: {item.name}"
                        ) from link_exc
+        for name in extra_templates:
+            (view / name).symlink_to(model_source / name)
        (view / ".modelhub-compat-report.json").write_text(
-            json.dumps({"source": str(source), "repairs": repairs}, ensure_ascii=False)
+            json.dumps(
+                {
+                    "model_source": str(model_source),
+                    "tokenizer_source": str(source),
+                    "repairs": repairs,
+                },
+                ensure_ascii=False,
+            )
            + "\n",
            encoding="utf-8",
        )
@@ -153,11 +186,13 @@ def prepare_model(model_dir: str | Path, *, work_root: str | Path | None = None)


 def main() -> int:
-    if len(sys.argv) != 2:
-        print("usage: engine_compat.py MODEL_DIR", file=sys.stderr)
+    if len(sys.argv) not in (2, 3):
+        print("usage: engine_compat.py MODEL_DIR [TOKENIZER_DIR]", file=sys.stderr)
        return 2
    try:
-        view = prepare_model(sys.argv[1])
+        view = prepare_model(
+            sys.argv[1], tokenizer_dir=sys.argv[2] if len(sys.argv) == 3 else None
+        )
    except CompatibilityError as exc:
        print(f"model compatibility error: {exc}", file=sys.stderr)
        return 2
--- a/tests/test_engine_compat.py
+++ b/tests/test_engine_compat.py
@@ -84,6 +84,47 @@ class EngineCompatTests(unittest.TestCase):
            )
            self.assertIsNone(prepare_model(source, work_root=root))

+    def test_explicit_tokenizer_directory_gets_qwen_template(self):
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            model = self._model(root, {"eos_token": "<|im_end|>"})
+            tokenizer = root / "vendor-tokenizer"
+            tokenizer.mkdir()
+            (tokenizer / "tokenizer_config.json").write_text(
+                json.dumps(
+                    {
+                        "eos_token": "<|im_end|>",
+                        "extra_special_tokens": ["<|im_start|>"],
+                        "tokenizer_class": "TokenizersBackend",
+                    }
+                ),
+                encoding="utf-8",
+            )
+            (tokenizer / "tokenizer.json").write_text("{}", encoding="utf-8")
+            view = Path(prepare_model(model, tokenizer_dir=tokenizer, work_root=root))
+            fixed = json.loads((view / "tokenizer_config.json").read_text("utf-8"))
+            self.assertEqual(fixed["chat_template"], QWEN_CHAT_TEMPLATE)
+            self.assertEqual(fixed["additional_special_tokens"], ["<|im_start|>"])
+            self.assertEqual(fixed["tokenizer_class"], "PreTrainedTokenizerFast")
+            self.assertEqual((model / "model.safetensors").read_bytes(), b"sample-weights")
+            self.assertFalse((view / "model.safetensors").exists())
+            self.assertEqual((view / "tokenizer.json").read_text("utf-8"), "{}")
+
+    def test_explicit_tokenizer_directory_preserves_model_standalone_template(self):
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            model = self._model(root, {"eos_token": "<|im_end|>"})
+            (model / "chat_template.jinja").write_text("{{ messages }}", encoding="utf-8")
+            tokenizer = root / "vendor-tokenizer"
+            tokenizer.mkdir()
+            (tokenizer / "tokenizer_config.json").write_text(
+                json.dumps({"eos_token": "<|im_end|>"}), encoding="utf-8"
+            )
+            view = Path(prepare_model(model, tokenizer_dir=tokenizer, work_root=root))
+            fixed = json.loads((view / "tokenizer_config.json").read_text("utf-8"))
+            self.assertNotIn("chat_template", fixed)
+            self.assertEqual((view / "chat_template.jinja").read_text("utf-8"), "{{ messages }}")
+

 if __name__ == "__main__":
    unittest.main()
--- a/vllm_wrapper.sh
+++ b/vllm_wrapper.sh
@@ -9,17 +9,39 @@ fi

 model_dir="$2"
 shift 2
-prepared="$(python3 /opt/modelhub/engine_compat.py "$model_dir")"
+args=("$@")
+tokenizer_dir=""
+for ((i=0; i<${#args[@]}; i++)); do
+    if [[ "${args[i]}" == "--tokenizer" && $((i+1)) -lt ${#args[@]} ]]; then
+        tokenizer_dir="${args[i+1]}"
+        break
+    elif [[ "${args[i]}" == --tokenizer=* ]]; then
+        tokenizer_dir="${args[i]#--tokenizer=}"
+        break
+    fi
+done
+
+if [[ -n "$tokenizer_dir" && -d "$tokenizer_dir" ]]; then
+    prepared="$(python3 /opt/modelhub/engine_compat.py "$model_dir" "$tokenizer_dir")"
+else
+    prepared="$(python3 /opt/modelhub/engine_compat.py "$model_dir")"
+    tokenizer_dir=""
+fi
 if [[ -z "$prepared" ]]; then
-    exec "$vllm_real" serve "$model_dir" "$@"
+    exec "$vllm_real" serve "$model_dir" "${args[@]}"
 fi

-args=("$@")
+effective_model="$prepared"
+if [[ -n "$tokenizer_dir" && "$(readlink -f "$tokenizer_dir")" != "$(readlink -f "$model_dir")" ]]; then
+    # A vendor entrypoint's --tokenizer points at a copied tokenizer tree.
+    # Keep model weights at the original path; redirect only that tokenizer.
+    effective_model="$model_dir"
+fi
 for ((i=0; i<${#args[@]}; i++)); do
-    if [[ "${args[i]}" == "--tokenizer" && $((i+1)) -lt ${#args[@]} && "${args[i+1]}" == "$model_dir" ]]; then
+    if [[ "${args[i]}" == "--tokenizer" && $((i+1)) -lt ${#args[@]} && "${args[i+1]}" == "$tokenizer_dir" ]]; then
        args[i+1]="$prepared"
-    elif [[ "${args[i]}" == "--tokenizer=$model_dir" ]]; then
+    elif [[ "${args[i]}" == "--tokenizer=$tokenizer_dir" && -n "$tokenizer_dir" ]]; then
        args[i]="--tokenizer=$prepared"
    fi
 done
-exec "$vllm_real" serve "$prepared" "${args[@]}"
+exec "$vllm_real" serve "$effective_model" "${args[@]}"