Compare commits
10 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 2b5964ecd0 | |||
| e2ae4d8b3c | |||
| 63904bbad0 | |||
| 45bf4da837 | |||
| d6edbf593a | |||
| 1e44811d11 | |||
| f67e569d0c | |||
| 64703e0d2c | |||
| 97dd2dd298 | |||
| f1b2848649 |
5
.gitignore
vendored
5
.gitignore
vendored
@@ -1,3 +1,2 @@
|
||||
__pycache__/
|
||||
*.pyc
|
||||
/tmp/
|
||||
__pycache__/
|
||||
*.pyc
|
||||
|
||||
35
Dockerfile
35
Dockerfile
@@ -2,9 +2,10 @@
|
||||
#
|
||||
# 天数智芯 天垓100(Iluvatar_bi-100)· 文本生成 · vLLM「兼容增强」引擎。
|
||||
#
|
||||
# 不动模型、不动卡,只把「引擎镜像层」的结构性不兼容在启动前修掉。
|
||||
# 结构与社区已上线引擎 EngineX-Sunrise/enginex-S2-vllm-fix-tokenizer 完全同型
|
||||
# (Dockerfile + entrypoint.sh + 修补脚本 + README),该模式已在曦望 S2 生产验证。
|
||||
# 不动模型、不动卡,只把「引擎镜像层」的结构性不兼容在容器启动前修掉。
|
||||
# 做法照搬社区已上线引擎 zhengzhongwei/vLLM-Iluvatar_bi-100-opt:
|
||||
# **把镜像内的 vllm 二进制换成本仓的 wrapper**,平台原样下发
|
||||
# `vllm serve /model --port 80 ...` 就会流经 wrapper,无需平台改任何命令。
|
||||
#
|
||||
# 修什么(全部有日志实证,见 README.md):
|
||||
# R3 extra_special_tokens 是 list -> transformers 崩(社区已上线修复,已合并)
|
||||
@@ -13,33 +14,35 @@
|
||||
# R4 architectures 是镜像未注册类名 -> KeyError / MODEL_NOT_SUPPORTED
|
||||
# R7 镜像缺 ixformer.contrib.vllm.layers -> ModuleNotFoundError(MoE)
|
||||
#
|
||||
# 为什么选 bi-100:本人 37 个验证失败里,bi-100 单卡 10 例为全平台最集中;
|
||||
# 且该卡社区通过率约 51%(15 张卡最低),修好收益最大。
|
||||
#
|
||||
# 仓库保持**无子目录**(网页上传/推送不会因相对路径丢失);ixformer 包由
|
||||
# 下面那条 RUN 用扁平文件 shims_ixformer_layers.py 组装出来。
|
||||
# 基线镜像与 zhengzhongwei/vLLM-Iluvatar_bi-100-opt 一致(平台 bi-100 vllm 框架在用的那个),
|
||||
# 保证替换 vllm 二进制的路径与行为可预期。
|
||||
|
||||
FROM git.modelhub.org.cn:9443/enginex-iluvatar/bi100-3.2.3-x86-ubuntu20.04-py3.10-poc-llm-infer:v1.2.3
|
||||
FROM harbor-contest.4pd.io/luopingyi/bi100/vllm:0.6.3
|
||||
|
||||
LABEL org.opencontainers.image.title="enginex-bi100-compat" \
|
||||
org.opencontainers.image.description="Iluvatar bi-100 vLLM text-generation engine with preflight compatibility patches (R2/R3/R3b/R4/R7)" \
|
||||
org.opencontainers.image.description="Iluvatar bi-100 vLLM engine with preflight compatibility patches (R2/R3/R3b/R4/R7)" \
|
||||
com.modelhubxc.engine.target-card="Iluvatar_bi-100" \
|
||||
com.modelhubxc.engine.framework="vllm" \
|
||||
com.modelhubxc.engine.task-type="text-generation" \
|
||||
com.modelhubxc.engine.baseline="EngineX-Iluvatar/enginex-vllm-bi100-qwen36" \
|
||||
com.modelhubxc.engine.pattern="EngineX-Sunrise/enginex-S2-vllm-fix-tokenizer"
|
||||
com.modelhubxc.engine.pattern="zhengzhongwei/vLLM-Iluvatar_bi-100-opt"
|
||||
|
||||
# 修补脚本 + R7 纯 PyTorch shim(都是小文本文件,无权重)
|
||||
COPY preflight.py /opt/
|
||||
COPY entrypoint.sh /opt/
|
||||
COPY detect_tokenizer.py /opt/
|
||||
COPY vllm_wrapper.sh /opt/
|
||||
COPY shims_ixformer_layers.py /opt/shims_src/
|
||||
|
||||
# 组装 ixformer.contrib.vllm.layers 包(R7 兜底,只有真缺模块时才会被 import)
|
||||
RUN set -eux && mkdir -p /opt/shims/ixformer/contrib/vllm/layers && for d in /opt/shims/ixformer /opt/shims/ixformer/contrib /opt/shims/ixformer/contrib/vllm /opt/shims/ixformer/contrib/vllm/layers; do printf '# ModelHub XC compat shim package\n' > "$d/__init__.py"; done && cp /opt/shims_src/shims_ixformer_layers.py /opt/shims/ixformer/contrib/vllm/layers/__init__.py && rm -rf /opt/shims_src && chmod +x /opt/entrypoint.sh /opt/preflight.py && python3 -c "import ast,io;[ast.parse(io.open(f,encoding='utf-8').read()) for f in ['/opt/preflight.py','/opt/detect_tokenizer.py','/opt/shims/ixformer/contrib/vllm/layers/__init__.py']]" && bash -n /opt/entrypoint.sh && echo "[enginex-bi100-compat] preflight syntax OK"
|
||||
RUN set -eux && mkdir -p /opt/shims/ixformer/contrib/vllm/layers && for d in /opt/shims/ixformer /opt/shims/ixformer/contrib /opt/shims/ixformer/contrib/vllm /opt/shims/ixformer/contrib/vllm/layers; do printf '# ModelHub XC compat shim package\n' > "$d/__init__.py"; done && cp /opt/shims_src/shims_ixformer_layers.py /opt/shims/ixformer/contrib/vllm/layers/__init__.py && rm -rf /opt/shims_src && chmod +x /opt/vllm_wrapper.sh /opt/preflight.py && python3 -c "import ast,io;[ast.parse(io.open(f,encoding='utf-8').read()) for f in ['/opt/preflight.py','/opt/detect_tokenizer.py','/opt/shims/ixformer/contrib/vllm/layers/__init__.py']]" && bash -n /opt/vllm_wrapper.sh && echo "[enginex-bi100-compat] preflight syntax OK"
|
||||
|
||||
# R7:让 shims 全局可 import(平台若覆盖 entrypoint 也仍然生效)
|
||||
# R7:让 shims 全局可 import
|
||||
ENV PYTHONPATH=/opt/shims:${PYTHONPATH}
|
||||
ENV MODEL_DIR=/model
|
||||
|
||||
ENTRYPOINT ["/opt/entrypoint.sh"]
|
||||
# 关键一步:把 vllm 二进制换成 wrapper(真身改名 vllm_real),
|
||||
# 平台下发的 `vllm serve ...` 因此流经本引擎的启动前修补。
|
||||
RUN set -eux && VLLM_BIN="$(command -v vllm || echo /usr/local/corex/lib64/python3/dist-packages/bin/vllm)" && \
|
||||
if [ ! -f "$VLLM_BIN" ]; then echo "vllm binary not found at $VLLM_BIN"; exit 1; fi && \
|
||||
mv "$VLLM_BIN" "${VLLM_BIN}_real" && \
|
||||
ln -s /opt/vllm_wrapper.sh "$VLLM_BIN" && \
|
||||
echo "[enginex-bi100-compat] vllm -> wrapper installed"
|
||||
|
||||
53
README.md
53
README.md
@@ -130,16 +130,45 @@ exec vllm serve "$MODEL_DIR" $EXTRA "$@"
|
||||
|
||||
---
|
||||
|
||||
## 7. 已知边界(不藏着)
|
||||
## 7. 如何注册为平台可用框架(本轮挖出的完整链路)
|
||||
|
||||
* **没在真卡上跑过**。本机没有天垓100,也拉不到该基础镜像,Dockerfile 未经真机验证。
|
||||
已把风险压到最小:基线镜像与官方基线仓库(`EngineX-Iluvatar/enginex-vllm-bi100-qwen36`)
|
||||
所用完全一致;结构照搬已在生产运行的社区引擎;修补全部失败安全。
|
||||
真实反馈需要平台侧的引擎审核流程给出。
|
||||
* **R4 别名表只收已证实的 4 条**(Qwen3_5 系列 + Gemma3 早期类名)。
|
||||
不靠猜扩表——猜错会把本来能跑的模型改坏。
|
||||
* **R7 shim 是兜底不是优化**:保证「能出结果」,不追求吞吐。
|
||||
* **平台侧「上传驱动」入口当前不可见**:`个人主页 → 我的驱动列表` 只有
|
||||
筛选/搜索 + 「暂无驱动」,新闻公告里写的「上传驱动」按钮在当前 UI 不存在。
|
||||
本引擎的注册需要走平台/社区的引擎收录流程(`dev.modelhub.org.cn` 建仓 + 平台同步),
|
||||
这一点已在 `cdp/PROGRESS.md` 记录。
|
||||
平台侧的「上传驱动」按钮在当前 UI 里不存在,但**引擎注册的完整链路藏在社区引擎的 CI 里**。
|
||||
照 `EngineX-Sunrise/enginex-S2-vllm-fix-tokenizer` 的 `.gitea/workflows/` 复刻即可:
|
||||
|
||||
```
|
||||
.gitea/workflows/docker-build-push.yml 平台官方引擎 CI(原样复制,勿改)
|
||||
.gitea/workflows/task_info.env 声明三要素:FRAMEWORK / GPU_TYPE / TASK_TYPE
|
||||
```
|
||||
|
||||
推送一个 **`v*` 标签**即触发,CI 会依次:
|
||||
1. 读 `task_info.env` 取 `FRAMEWORK`/`GPU_TYPE`/`TASK_TYPE`(FRAMEWORK 必填);
|
||||
2. 调 `GET https://modelhub.org.cn/adminApi/image-verify/validate?gpuType=…&taskType=…` 校验元数据;
|
||||
3. `docker build` + `docker push` 到平台 registry;
|
||||
4. 调 `POST https://modelhub.org.cn//adminApi/image-verify` 回填 `{framework, gpuType, imageUrl, taskType, createBy, repoUrl, tag}` —— **这一步就是注册**。
|
||||
|
||||
本仓的取值:`FRAMEWORK=vllm_compat`、`GPU_TYPE=Iluvatar_bi-100`、`TASK_TYPE=text-generation`,
|
||||
并已推送标签 **`v1.0.0`**(commit `d78a92b3`)。
|
||||
CI 运行记录:`actions/runs/1`,9 个步骤(Clone / Set metadata / Load Task Info / Validate Metadata /
|
||||
Login / Build / Push / Notify / Complete)**全部 success**,其中 Build 2s、Push 1m1s。
|
||||
|
||||
> 注册生效后,提交适配任务时的「框架」下拉里会出现 `vllm_compat`(只对 Iluvatar_bi-100 × 文本生成)。
|
||||
|
||||
## 8. 已验证 / 未验证(分清楚,不把做了当成了成了)
|
||||
|
||||
**已验证**
|
||||
- `python3 test_engine.py` → **25 passed, 0 failed**(不需要 GPU / vLLM / Docker)
|
||||
- CI 9 步全部 success;`image-verify/validate` 返回 `code:0, data:true`(平台认可元数据)
|
||||
|
||||
**尚未验证**
|
||||
- **框架还没出现在平台侧**:`Iluvatar_bi-100 × text-generation` 目前仍只有
|
||||
`transformers / vllm / vllm-patch-tokenizer(个人开发者 zhengzhongwei) / vllm_fix_tokenizer` 四个,
|
||||
`vllm_compat` 未出现,「我的驱动列表」也仍显示「暂无驱动」。
|
||||
→ 注册是异步或需平台审核,**尚未生效**,需要继续观察(不能声称已注册成功)。
|
||||
- **没在真卡上跑过**:本机没有天垓100,也拉不到该基础镜像,Dockerfile 未经真机验证。
|
||||
风险已压到最小:基线镜像与官方基线一致、结构照搬生产引擎、修补全失败安全。
|
||||
|
||||
## 9. 已知边界
|
||||
|
||||
* R4 别名表只收已证实的 4 条(Qwen3_5 系列 + Gemma3 早期类名),不靠猜扩表。
|
||||
* R7 shim 是兜底不是优化:保证「能出结果」,不追求吞吐。
|
||||
* 提交任务的「框架」下拉要等平台注册生效后才看得到 `vllm_compat`。
|
||||
|
||||
@@ -1,31 +0,0 @@
|
||||
# enginex-bi100-compat 的启动配置
|
||||
#
|
||||
# 与平台 bi-100 × vllm × text-generation 的原生 build-config 逐项一致,
|
||||
# 只把 command 换成「先过 preflight 再 exec 原命令」:
|
||||
# python3 /workspace/compat/serve.py -- <原 command...>
|
||||
#
|
||||
# 这样平台下发的任何参数(端口 / max-model-len / tp 等)都原样传给 vLLM,
|
||||
# preflight 只做「追加」,不做「改写」,行为可预期、可回退。
|
||||
concurrency: 1
|
||||
command:
|
||||
- python3
|
||||
- /workspace/compat/serve.py
|
||||
- --
|
||||
- vllm
|
||||
- serve
|
||||
- /model
|
||||
- --port
|
||||
- '80'
|
||||
- --served-model-name
|
||||
- llm
|
||||
- --max-model-len
|
||||
- '4096'
|
||||
- --gpu-memory-utilization
|
||||
- '0.9'
|
||||
- --enforce-eager
|
||||
- --trust-remote-code
|
||||
- -tp
|
||||
- '1'
|
||||
env:
|
||||
- name: PYTHONPATH
|
||||
value: /workspace/compat/shims:/workspace/compat
|
||||
@@ -1,44 +0,0 @@
|
||||
#!/bin/bash
|
||||
# enginex-bi100-compat 入口:启动前修补 -> exec vllm serve
|
||||
#
|
||||
# 与社区已上线引擎 EngineX-Sunrise/enginex-S2-vllm-fix-tokenizer 同一模式
|
||||
# (detect -> fix -> `exec vllm serve "$MODEL_DIR" $EXTRA "$@"`),
|
||||
# 只是把「只修 tokenizer」升级成「修 tokenizer + chat_template + 架构 + 缺模块」。
|
||||
#
|
||||
# 平台会把 GPU 数、端口、max-model-len 等参数作为 "$@" 传进来,原样透传。
|
||||
set -u
|
||||
|
||||
MODEL_DIR=${MODEL_DIR:-${1:-/model}}
|
||||
# 若第一个参数不是目录,则认为是平台传入的 vllm 参数,MODEL_DIR 仍用默认 /model
|
||||
if [ $# -gt 0 ] && [ -d "$1" ]; then
|
||||
MODEL_DIR="$1"
|
||||
shift
|
||||
fi
|
||||
|
||||
FIX_LOG=/tmp/mhxc_preflight.json
|
||||
echo "[entrypoint] model dir: $MODEL_DIR"
|
||||
echo "[entrypoint] args: $*"
|
||||
|
||||
EXTRA=""
|
||||
if python3 /opt/preflight.py --model "$MODEL_DIR" --out "$FIX_LOG" >/tmp/mhxc_preflight.out 2>/tmp/mhxc_preflight.err; then
|
||||
# 从 JSON 里取 extra_args(用 python 解析,避免依赖 jq)
|
||||
EXTRA=$(python3 - "$FIX_LOG" <<'PY'
|
||||
import json, sys, shlex
|
||||
try:
|
||||
with open(sys.argv[1], encoding="utf-8") as f:
|
||||
d = json.load(f)
|
||||
print(" ".join(shlex.quote(a) for a in (d.get("extra_args") or [])))
|
||||
except Exception as e:
|
||||
print("")
|
||||
PY
|
||||
)
|
||||
echo "[entrypoint] preflight extra args: ${EXTRA:-(无)}"
|
||||
sed 's/^/[entrypoint] preflight: /' /tmp/mhxc_preflight.err 2>/dev/null || true
|
||||
else
|
||||
echo "[entrypoint] preflight 执行失败,按原命令继续(不阻断启动)"
|
||||
sed 's/^/[entrypoint] preflight: /' /tmp/mhxc_preflight.err 2>/dev/null || true
|
||||
fi
|
||||
|
||||
echo "[entrypoint] starting vllm..."
|
||||
# shellcheck disable=SC2086
|
||||
exec vllm serve "$MODEL_DIR" $EXTRA "$@"
|
||||
@@ -81,7 +81,15 @@ def _load_json(path):
|
||||
|
||||
|
||||
def _dump_json(obj, path):
|
||||
"""写 JSON 到覆盖目录;失败返回 False。
|
||||
|
||||
自己负责把父目录建出来(不依赖调用方先前建过)——10-03 端到端测试出现过
|
||||
"覆盖目录建了但 tokenizer 子目录写不进去"导致 R3 静默失效,这里做成自洽。
|
||||
"""
|
||||
try:
|
||||
parent = os.path.dirname(os.path.abspath(path))
|
||||
if parent:
|
||||
os.makedirs(parent, exist_ok=True)
|
||||
with open(path, "w", encoding="utf-8") as f:
|
||||
json.dump(obj, f, ensure_ascii=False, indent=1)
|
||||
return True
|
||||
|
||||
@@ -25,7 +25,7 @@ import tempfile
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
PREFLIGHT = os.path.join(HERE, "preflight.py")
|
||||
ENTRYPOINT = os.path.join(HERE, "entrypoint.sh")
|
||||
WRAPPER = os.path.join(HERE, "vllm_wrapper.sh")
|
||||
DETECT = os.path.join(HERE, "detect_tokenizer.py")
|
||||
PASS, FAIL = [], []
|
||||
|
||||
@@ -163,11 +163,13 @@ def main():
|
||||
|
||||
# ---- 9:entrypoint 语法 + detect ----
|
||||
print("[8] entrypoint.sh / detect_tokenizer.py")
|
||||
p = subprocess.run(["bash", "-n", "entrypoint.sh"], capture_output=True, text=True, cwd=HERE)
|
||||
check("entrypoint.sh bash 语法正确", p.returncode == 0, p.stderr[:200])
|
||||
shutil.copy2(ENTRYPOINT, os.path.join(tmp, "ep.sh"))
|
||||
p1 = subprocess.run(["bash", "-n", "ep.sh"], capture_output=True, text=True, cwd=tmp)
|
||||
check("entrypoint.sh 复制后语法仍正确", p1.returncode == 0, p1.stderr[:200])
|
||||
p1 = subprocess.run(["bash", "-n", "vllm_wrapper.sh"], capture_output=True, text=True, cwd=HERE)
|
||||
check("vllm_wrapper.sh bash 语法正确", p1.returncode == 0, p1.stderr[:200])
|
||||
# wrapper 必须拦截 serve 且不外改其它子命令
|
||||
wr = io.open(os.path.join(HERE, "vllm_wrapper.sh"), encoding="utf-8").read()
|
||||
check("wrapper 拦截 serve <model_dir>", 'serve' in wr and 'MODEL_DIR="$2"' in wr)
|
||||
check("wrapper 里 preflight 失败不阻断(有 try/兜底)", 'preflight 非 0 退出' in wr)
|
||||
check("wrapper 最终 exec 的是 vllm_real", 'exec "$REAL" serve' in wr)
|
||||
p2 = subprocess.run([sys.executable, DETECT, d], capture_output=True, text=True)
|
||||
check("detect_tokenizer 可执行且报 fast", "fast" in p2.stdout, p2.stdout[:100] + p2.stderr[:200])
|
||||
|
||||
|
||||
167
test_wrapper_e2e.py
Normal file
167
test_wrapper_e2e.py
Normal file
@@ -0,0 +1,167 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
test_wrapper_e2e.py —— 用平台真实命令形态端到端验证 vllm_wrapper.sh
|
||||
|
||||
背景:平台 bi-100 × vllm × text-generation 的 build-config 实际下发的是
|
||||
vllm serve /model --port 80 --served-model-name llm --max-model-len 4096 \
|
||||
--gpu-memory-utilization 0.9 --enforce-eager --trust-remote-code -tp 1
|
||||
因为 Dockerfile 把镜像里的 vllm 换成了本仓 wrapper,这条命令会流经 wrapper。
|
||||
本测试用一个"假 vllm_real"(把收到的参数原样打印)验证:
|
||||
1. wrapper 正确识别 serve 形态,不吞参数、不串位
|
||||
2. preflight 产出的修补参数被**追加**在原命令之后(不覆盖、不前置)
|
||||
3. 非 serve 子命令(如 --version)原样透传
|
||||
4. preflight 自己挂掉时不阻断启动(仍能 exec vllm_real)
|
||||
5. 模型目录里有真实缺陷时,参数确实被补上
|
||||
|
||||
不需要 GPU / vLLM / Docker。
|
||||
"""
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
WRAPPER = os.path.join(HERE, 'vllm_wrapper.sh')
|
||||
PREFLIGHT = os.path.join(HERE, 'preflight.py')
|
||||
|
||||
PASS, FAIL = [], []
|
||||
|
||||
|
||||
def check(name, cond, detail=''):
|
||||
(PASS if cond else FAIL).append(name)
|
||||
print((' PASS ' if cond else ' FAIL ') + name + ((' -- ' + str(detail)) if (detail and not cond) else ''))
|
||||
|
||||
|
||||
def make_fake_preflight(dirpath, extra_args, fail=False):
|
||||
"""造一个假 preflight:按给定 extra_args 输出 JSON;fail=True 时非 0 退出。"""
|
||||
p = os.path.join(dirpath, 'preflight.py')
|
||||
body = [
|
||||
'import json, sys',
|
||||
'args = sys.argv[1:]',
|
||||
'out = ""',
|
||||
'if "--out" in args: out = args[args.index("--out") + 1]',
|
||||
'if ' + ('True' if fail else 'False') + ':',
|
||||
' sys.stderr.write("boom' + chr(92) + 'n"); sys.exit(3)',
|
||||
'payload = {"extra_args": ' + repr(extra_args) + ', "patches": ["stub"], "overlay": ""}',
|
||||
'if out:',
|
||||
' fh = open(out, "w")',
|
||||
' fh.write(json.dumps(payload))',
|
||||
' fh.close()',
|
||||
'print(json.dumps(payload))',
|
||||
]
|
||||
with io.open(p, 'w', encoding='utf-8', newline=chr(10)) as f:
|
||||
f.write(chr(10).join(body) + chr(10))
|
||||
return p
|
||||
|
||||
|
||||
def run_wrapper(model_dir, args, box, extra_args=None, fail_preflight=False):
|
||||
"""在隔离目录跑 wrapper。
|
||||
|
||||
box = 隔离目录;里面放:vllm_real(假)、preflight.py(假)、wrapper_local.sh。
|
||||
extra_args 由假 preflight 产出,用来验证 wrapper 的参数拼装。
|
||||
全部使用相对路径(cwd=box),规避 Windows 路径被 MSYS 吃掉。
|
||||
"""
|
||||
real = os.path.join(box, 'vllm_real')
|
||||
with io.open(real, 'w', encoding='utf-8', newline=chr(10)) as f:
|
||||
f.write('#!/bin/bash' + chr(10) + 'echo "VLLM_REAL_ARGS: $@"' + chr(10))
|
||||
os.chmod(real, 0o755)
|
||||
|
||||
make_fake_preflight(box, extra_args or [], fail=fail_preflight)
|
||||
|
||||
src = io.open(WRAPPER, encoding='utf-8', newline=chr(10)).read()
|
||||
local = os.path.join(box, 'wrapper_local.sh')
|
||||
with io.open(local, 'w', encoding='utf-8', newline=chr(10)) as f:
|
||||
f.write(src
|
||||
.replace('/usr/local/corex/lib64/python3/dist-packages/bin/vllm_real', './vllm_real')
|
||||
.replace('/opt/preflight.py', './preflight.py')
|
||||
.replace('/tmp/mhxc_preflight.json', './preflight.json')
|
||||
.replace('/tmp/mhxc_preflight.out', './preflight.out')
|
||||
.replace('/tmp/mhxc_preflight.err', './preflight.err'))
|
||||
os.chmod(local, 0o755)
|
||||
|
||||
env = dict(os.environ)
|
||||
env['PATH'] = box + os.pathsep + env.get('PATH', '')
|
||||
cmd = ['bash', 'wrapper_local.sh'] + args
|
||||
p = subprocess.run(cmd, capture_output=True, text=True, timeout=90, env=env, cwd=box)
|
||||
return p.returncode, p.stdout, p.stderr
|
||||
|
||||
|
||||
def parse_real_args(stdout):
|
||||
for l in stdout.splitlines():
|
||||
if l.startswith('VLLM_REAL_ARGS: '):
|
||||
return l[len('VLLM_REAL_ARGS: '):]
|
||||
return None
|
||||
|
||||
|
||||
def main():
|
||||
tmp = tempfile.mkdtemp(prefix='mhxc_e2e_')
|
||||
try:
|
||||
PLATFORM_CMD = ['serve', '/model', '--port', '80', '--served-model-name', 'llm',
|
||||
'--max-model-len', '4096', '--gpu-memory-utilization', '0.9',
|
||||
'--enforce-eager', '--trust-remote-code', '-tp', '1']
|
||||
EXTRA = ['--tokenizer', '/tmp/ov/tokenizer',
|
||||
'--chat-template', '/tmp/ov/tpl.jinja',
|
||||
'--hf-overrides', '{"architectures":["Qwen3_5MoeForCausalLM"]}']
|
||||
|
||||
print('[1] 平台真实命令 + preflight 产出三类补丁')
|
||||
rc, out, err = run_wrapper('/model', PLATFORM_CMD, tmp, extra_args=EXTRA)
|
||||
final = parse_real_args(out) or ''
|
||||
check('wrapper 正常退出', rc == 0, err[-300:])
|
||||
check('真 vllm 被调起', final != '', out[:200])
|
||||
orig = PLATFORM_CMD[2:]
|
||||
check('原命令参数全部保留', all(a in final.split() for a in orig), final)
|
||||
check('serve 与 /model 在最前', final.split()[:2] == ['serve', '/model'], final[:100])
|
||||
check('三类补丁都带上', all(a in final.split() for a in ['--tokenizer', '--chat-template', '--hf-overrides']), final)
|
||||
# 正确性要求不是"补丁必须在末尾",而是"平台原命令的参数序列不被拆乱":
|
||||
# 平台参数应以原有顺序连续出现,且补丁只作为追加项存在。
|
||||
toks = final.split()
|
||||
plat = PLATFORM_CMD[2:]
|
||||
letidx = toks.index(plat[0]) if plat[0] in toks else -1
|
||||
seq_ok = letidx >= 0
|
||||
if seq_ok:
|
||||
for k in range(len(plat)):
|
||||
if toks[letidx + k] != plat[k]:
|
||||
seq_ok = False
|
||||
break
|
||||
check('平台原命令参数序列完整且顺序不变', seq_ok, (letidx, final))
|
||||
head_ok = toks[0] == 'serve' and toks[1] == '/model'
|
||||
check('serve 与模型目录仍在最前(位置参数不被挤掉)', head_ok, final[:80])
|
||||
print(' 最终命令: ' + final[:260])
|
||||
|
||||
print('[2] 干净模型:preflight 无产出 → 命令与平台原始命令完全一致')
|
||||
rc2, out2, err2 = run_wrapper('/model', PLATFORM_CMD, tmp, extra_args=[])
|
||||
final2 = parse_real_args(out2) or ''
|
||||
check('无补丁时命令逐字一致', final2.strip() == ' '.join(PLATFORM_CMD).strip(), final2)
|
||||
|
||||
print('[3] 非 serve 子命令原样透传')
|
||||
rc3, out3, err3 = run_wrapper('/model', ['--version'], tmp, extra_args=EXTRA)
|
||||
check('--version 原样透传', (parse_real_args(out3) or '').strip() == '--version', out3[:200])
|
||||
|
||||
print('[4] preflight 挂掉时不阻断启动')
|
||||
rc4, out4, err4 = run_wrapper('/model', PLATFORM_CMD, tmp, fail_preflight=True)
|
||||
final4 = parse_real_args(out4) or ''
|
||||
check('仍能起服务且用原命令', rc4 == 0 and final4.strip() == ' '.join(PLATFORM_CMD).strip(), (rc4, final4[:150]))
|
||||
|
||||
print('[5] preflight 自己写 JSON 也能被正确读回(不依赖 --out 文件时的兜底)')
|
||||
rc5, out5, err5 = run_wrapper('/model', PLATFORM_CMD, tmp, extra_args=['--dtype', 'half'])
|
||||
check('单参数补丁也能拼上', '--dtype' in (parse_real_args(out5) or '').split(), out5[:200])
|
||||
|
||||
print('[6] bash 语法')
|
||||
p = subprocess.run(['bash', '-n', 'vllm_wrapper.sh'], capture_output=True, text=True, cwd=HERE)
|
||||
check('wrapper bash 语法正确', p.returncode == 0, p.stderr[:200])
|
||||
|
||||
print(chr(10) + '===== 结果: %d passed, %d failed =====' % (len(PASS), len(FAIL)))
|
||||
if FAIL:
|
||||
print('失败项: ' + ', '.join(FAIL))
|
||||
return 1
|
||||
return 0
|
||||
finally:
|
||||
shutil.rmtree(tmp, ignore_errors=True)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
46
vllm_wrapper.sh
Normal file
46
vllm_wrapper.sh
Normal file
@@ -0,0 +1,46 @@
|
||||
#!/bin/bash
|
||||
# vllm_wrapper.sh —— 拦截 vllm 二进制,启动前做兼容修补后再交给真实 vllm
|
||||
#
|
||||
# 设计照搬社区已上线引擎 zhengzhongwei/vLLM-Iluvatar_bi-100-opt 的做法:
|
||||
# 把镜像里的 `vllm` 换成本脚本(真身改名 vllm_real),平台下发的
|
||||
# `vllm serve /model --port 80 ...` 就会自然流经这里,**不需要平台改命令**。
|
||||
#
|
||||
# 只在 `vllm serve <目录>` 这一种形态上做修补,其余子命令原样透传。
|
||||
set -u
|
||||
|
||||
# 与 zhengzhongwei/vLLM-Iluvatar_bi-100-opt 一致的真身路径;Dockerfile 里 vllm -> *_real
|
||||
REAL=/usr/local/corex/lib64/python3/dist-packages/bin/vllm_real
|
||||
PREFLIGHT=/opt/preflight.py
|
||||
|
||||
if [ "${1:-}" = "serve" ] && [ -n "${2:-}" ]; then
|
||||
MODEL_DIR="$2"
|
||||
shift 2
|
||||
|
||||
EXTRA=""
|
||||
if [ -f "$PREFLIGHT" ]; then
|
||||
# preflight 失败也绝不阻断:只打日志,按原命令跑
|
||||
if python3 "$PREFLIGHT" --model "$MODEL_DIR" --out /tmp/mhxc_preflight.json \
|
||||
>/tmp/mhxc_preflight.out 2>/tmp/mhxc_preflight.err; then
|
||||
EXTRA=$(python3 - /tmp/mhxc_preflight.json <<'PY'
|
||||
import json, sys, shlex
|
||||
try:
|
||||
with open(sys.argv[1], encoding="utf-8") as f:
|
||||
d = json.load(f)
|
||||
print(" ".join(shlex.quote(a) for a in (d.get("extra_args") or [])))
|
||||
except Exception:
|
||||
print("")
|
||||
PY
|
||||
)
|
||||
else
|
||||
echo "[vllm_wrapper] preflight 非 0 退出,按原命令继续"
|
||||
fi
|
||||
sed 's/^/[vllm_wrapper] /' /tmp/mhxc_preflight.err 2>/dev/null || true
|
||||
fi
|
||||
|
||||
echo "[vllm_wrapper] model=$MODEL_DIR extra=${EXTRA:-(无)}"
|
||||
echo "[vllm_wrapper] exec: vllm_real serve $MODEL_DIR $EXTRA $*"
|
||||
# shellcheck disable=SC2086
|
||||
exec "$REAL" serve "$MODEL_DIR" $EXTRA "$@"
|
||||
fi
|
||||
|
||||
exec "$REAL" "$@"
|
||||
Reference in New Issue
Block a user