356 lines
15 KiB
Python
356 lines
15 KiB
Python
#!/usr/bin/env python3
|
||
"""检测并修复 tokenizer_config.json 里与 vLLM 0.9.1(transformers 4.x)不兼容的写法。
|
||
|
||
面向 MetaX C500:容器启动时由 `vllm_wrapper.sh` 调用,
|
||
把模型目录里的 tokenizer 相关文件拷到临时目录、改好配置后用 `--tokenizer` 指过去,
|
||
**原始模型目录不做任何修改**。
|
||
|
||
## 修什么
|
||
|
||
| # | 缺陷 | 现场报错(vLLM 0.9.1 / transformers 4.5x) |
|
||
|---|---|---|
|
||
| 1 | `tokenizer_class` 不是 transformers 合法类名(典型 `TokenizersBackend`,新版本 transformers 写出) | `ValueError: Tokenizer class X does not exist or is not currently imported.` |
|
||
| 2 | `extra_special_tokens` 写成 list | `AttributeError: 'list' object has no attribute 'keys'` |
|
||
| 3 | 有 `tokenizer.json` 但 `from_slow: true` | `ValueError: Cannot instantiate this tokenizer from a slow version…`(transformers 会绕开 tokenizer.json 走 slow 路径,而 `PreTrainedTokenizerFast` 没有 slow 类) |
|
||
| 4 | 带 `auto_map` 的仓库自带 tokenizer 代码(`QWenTokenizer` / `MuPTTokenizer` / `InternLM2Tokenizer` …) | 改写 `tokenizer_class` 后 `auto_map` 仍生效,但 `/tmp/fixed_tokenizer` 里没有 `tokenization_*.py`:`OSError: … does not appear to have a file named tokenization_qwen.py` |
|
||
|
||
第 4 类**不能靠改写类名解决**:这些 tokenizer 是仓库自带 Python 代码实现的(还常带私有词表,
|
||
如 Qwen1 的 `qwen.tiktoken`),换成 `GPT2TokenizerFast` 之类的通用类会**改变分词结果**
|
||
(实测 MuPT 对 `<|endoftext|>` 的编码就不一样)。所以正确做法是把代码和词表一起拷过去、
|
||
保留 `auto_map`,让仓库自己的 tokenizer 跑起来。
|
||
|
||
## 怎么保证不把「能起」改成「起不来」
|
||
|
||
候选配置按「改动最小 → 逐级兜底」排队,**每个候选都真的用 `AutoTokenizer.from_pretrained(dst)`
|
||
试加载一次**(走 encode 一遍),第一个成功的才落到输出目录;全都失败就什么都不输出,
|
||
wrapper 于是按原样启动 vLLM —— 等价于没有这个补丁,绝不会更差。
|
||
|
||
候选顺序:
|
||
1. **最小改动**:只修 `extra_special_tokens` / `from_slow`,`tokenizer_class` 和 `auto_map` 原样保留
|
||
(类名合法、或 `auto_map` 指向的 `.py` 已随文件拷过去时可选);
|
||
2. **改写类名**:按目录里的数据文件挑 `PreTrainedTokenizerFast` / `LlamaTokenizer` / `GPT2TokenizerFast`,
|
||
同时去掉 `auto_map`(否则动态加载会去找已不存在的 `.py`)。
|
||
|
||
## 输出契约(wrapper 依赖)
|
||
|
||
* stdout:**只有一行**——可用的修复目录(没有就什么都不输出,退出码 0);
|
||
* stderr:`[fix_tokenizer]` 开头的诊断日志;
|
||
* 任何异常都吞掉并返回 0(不让补丁本身把容器搞挂)。
|
||
|
||
环境变量:
|
||
* `MODEL_DIR`:模型目录(也可用第一个位置参数);
|
||
* `FIX_TOKENIZER_OUT_DIR`:输出目录,默认 `/tmp/fixed_tokenizer`;
|
||
* `FIX_TOKENIZER_VERIFY=0`:跳过试加载(不建议;离线调试用);
|
||
* `FIX_TOKENIZER_MAX_COPY_MB`:单个文件的拷贝上限,默认 256(防止把权重拷进去)。
|
||
|
||
适配 MetaX C500 / vLLM 0.9.1 环境。
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import json
|
||
import os
|
||
import shutil
|
||
import sys
|
||
import traceback
|
||
from typing import Dict, Iterable, List, Optional, Tuple
|
||
|
||
DEFAULT_OUT_DIR = "/tmp/fixed_tokenizer"
|
||
|
||
# 权重容器:一律不拷(tokenizer 目录用不到,模型动辄几十 GB)
|
||
SKIP_SUFFIXES = (
|
||
".safetensors", ".bin", ".pt", ".pth", ".ckpt", ".gguf", ".onnx",
|
||
".msgpack", ".h5", ".npz", ".npy", ".pkl", ".sqlite", ".db",
|
||
)
|
||
|
||
# transformers 5.x 才认识的键,4.x 里没有意义;留着会被原样透传给 tokenizer 构造函数
|
||
V5_ONLY_KEYS = ("from_slow", "backend")
|
||
|
||
|
||
def log(msg: str) -> None:
|
||
print(f"[fix_tokenizer] {msg}", file=sys.stderr, flush=True)
|
||
|
||
|
||
# ── transformers 探测 ──────────────────────────────────────────────
|
||
|
||
|
||
def _transformers():
|
||
try:
|
||
import transformers # noqa: F401
|
||
except Exception: # pragma: no cover - 镜像里一定有
|
||
return None
|
||
return transformers
|
||
|
||
|
||
def class_exists(name: str) -> Optional[bool]:
|
||
"""类在 transformers 里是否存在。None = 判断不了(没装 transformers)。"""
|
||
tf = _transformers()
|
||
if tf is None or not name:
|
||
return None
|
||
return getattr(tf, name, None) is not None
|
||
|
||
|
||
# ── 文件 ──────────────────────────────────────────────────────────
|
||
|
||
|
||
def payload_files(model_dir: str, max_bytes: int) -> List[str]:
|
||
"""模型目录里值得拷进修复目录的文件(tokenizer 数据 + 仓库自带代码)。"""
|
||
out: List[str] = []
|
||
try:
|
||
names = sorted(os.listdir(model_dir))
|
||
except OSError:
|
||
return out
|
||
for name in names:
|
||
if name.startswith("."):
|
||
continue
|
||
if name.endswith(SKIP_SUFFIXES):
|
||
continue
|
||
src = os.path.join(model_dir, name)
|
||
if not os.path.isfile(src):
|
||
continue
|
||
try:
|
||
if os.path.getsize(src) > max_bytes:
|
||
continue
|
||
except OSError:
|
||
continue
|
||
out.append(name)
|
||
return out
|
||
|
||
|
||
def copy_payload(model_dir: str, dst: str, names: Iterable[str]) -> None:
|
||
os.makedirs(dst, exist_ok=True)
|
||
for name in names:
|
||
shutil.copy2(os.path.join(model_dir, name), os.path.join(dst, name))
|
||
|
||
|
||
def read_config(path: str) -> Optional[Dict]:
|
||
try:
|
||
with open(path, "r", encoding="utf-8") as fh:
|
||
data = json.load(fh)
|
||
except (OSError, ValueError):
|
||
return None
|
||
return data if isinstance(data, dict) else None
|
||
|
||
|
||
def write_config(path: str, cfg: Dict) -> None:
|
||
with open(path, "w", encoding="utf-8") as fh:
|
||
json.dump(cfg, fh, indent=2, ensure_ascii=False)
|
||
|
||
|
||
# ── 配置分析与候选 ────────────────────────────────────────────────
|
||
|
||
|
||
def auto_map_modules(cfg: Dict) -> List[str]:
|
||
"""`auto_map` 里 AutoTokenizer 指向的模块文件名(如 tokenization_qwen.py)。"""
|
||
am = cfg.get("auto_map")
|
||
entry = None
|
||
if isinstance(am, dict):
|
||
entry = am.get("AutoTokenizer")
|
||
elif isinstance(am, (list, tuple)): # 老格式:["tokenization_x.X", null]
|
||
entry = am
|
||
if not entry:
|
||
return []
|
||
refs = entry if isinstance(entry, (list, tuple)) else [entry]
|
||
mods: List[str] = []
|
||
for ref in refs:
|
||
if not isinstance(ref, str) or not ref:
|
||
continue
|
||
mod = ref.split(".")[0].strip()
|
||
if mod and f"{mod}.py" not in mods:
|
||
mods.append(f"{mod}.py")
|
||
return mods
|
||
|
||
|
||
def choose_class(files: Iterable[str]) -> Optional[str]:
|
||
"""按目录里实际存在的数据文件挑一个能实例化的 transformers 类。"""
|
||
names = set(files)
|
||
if "tokenizer.json" in names:
|
||
return "PreTrainedTokenizerFast"
|
||
if "tokenizer.model" in names:
|
||
return "LlamaTokenizer"
|
||
if "vocab.json" in names and "merges.txt" in names:
|
||
return "GPT2TokenizerFast"
|
||
return None
|
||
|
||
|
||
def apply_common_fixes(cfg: Dict, files: Iterable[str], force_fast: bool) -> Tuple[Dict, List[str]]:
|
||
"""两类「无条件该修」的项:extra_special_tokens list→dict、会误伤 tokenizer.json 的 from_slow。"""
|
||
new = json.loads(json.dumps(cfg)) # 深拷贝,不改入参
|
||
fixes: List[str] = []
|
||
names = set(files)
|
||
|
||
est = new.get("extra_special_tokens")
|
||
if isinstance(est, list):
|
||
new["extra_special_tokens"] = {str(t): str(t) for t in est}
|
||
fixes.append(f"extra_special_tokens: list({len(est)}) -> dict")
|
||
|
||
if new.get("from_slow") and (force_fast or "tokenizer.json" in names):
|
||
# from_slow=true 时 transformers 会丢弃 tokenizer.json 走 slow 转换;
|
||
# 目标类若是通用 fast 类(没有 slow_tokenizer_class)就会直接抛 slow-version 错误。
|
||
new.pop("from_slow", None)
|
||
fixes.append("from_slow: true dropped (use tokenizer.json)")
|
||
|
||
return new, fixes
|
||
|
||
|
||
def build_candidates(cfg: Dict, files: List[str]) -> List[Tuple[str, Dict, bool]]:
|
||
"""返回 [(说明, 候选配置, 是否需要 trust_remote_code)],按「改动最小」排序。"""
|
||
names = set(files)
|
||
cls = cfg.get("tokenizer_class") or ""
|
||
known = class_exists(cls) if cls else False
|
||
mods = auto_map_modules(cfg)
|
||
remote_ready = bool(mods) and all(m in names for m in mods)
|
||
out: List[Tuple[str, Dict, bool]] = []
|
||
|
||
# 候选 1:尽量不动,只做必改项;auto_map 的代码已经拷过去,让仓库自己的 tokenizer 跑
|
||
if known or remote_ready:
|
||
cand, fixes = apply_common_fixes(cfg, names, force_fast=False)
|
||
note = "keep-class"
|
||
if remote_ready:
|
||
note += "+auto_map"
|
||
out.append((note, cand, remote_ready))
|
||
|
||
# 候选 2:改写类名 + 去掉 auto_map(最后兜底,可能改变分词口径,日志里会警告)
|
||
replacement = choose_class(names)
|
||
if replacement and replacement != cls:
|
||
cand, fixes = apply_common_fixes(cfg, names, force_fast=True)
|
||
cand["tokenizer_class"] = replacement
|
||
cand.pop("auto_map", None)
|
||
if fixes:
|
||
log("rewrite-class fixes: " + "; ".join(fixes))
|
||
out.append((f"rewrite-class->{replacement}", cand, False))
|
||
|
||
return out
|
||
|
||
|
||
# ── 试加载 ────────────────────────────────────────────────────────
|
||
|
||
|
||
def try_load(directory: str, trust_remote_code: Optional[bool]) -> Optional[str]:
|
||
"""真的加载一次 tokenizer 并 encode,成功返回 None,失败返回原因。"""
|
||
try:
|
||
from transformers import AutoTokenizer
|
||
except Exception as exc: # pragma: no cover - 镜像里一定有
|
||
return f"transformers unavailable: {exc}"
|
||
try:
|
||
tok = AutoTokenizer.from_pretrained(directory, trust_remote_code=trust_remote_code)
|
||
tok.encode("hello world")
|
||
return None
|
||
except Exception as exc:
|
||
first = str(exc).splitlines()[0] if str(exc) else type(exc).__name__
|
||
return f"{type(exc).__name__}: {first[:180]}"
|
||
|
||
|
||
# ── 主流程 ────────────────────────────────────────────────────────
|
||
|
||
|
||
def fix(model_dir: str, out_dir: str, verify: bool, max_bytes: int, dry_run: bool) -> str:
|
||
cfg_path = os.path.join(model_dir, "tokenizer_config.json")
|
||
cfg = read_config(cfg_path)
|
||
if cfg is None:
|
||
log(f"no readable tokenizer_config.json at {cfg_path}, skip")
|
||
return ""
|
||
|
||
files = payload_files(model_dir, max_bytes)
|
||
if not files:
|
||
log("no copyable tokenizer files in model dir, skip")
|
||
return ""
|
||
|
||
cls = cfg.get("tokenizer_class") or ""
|
||
known = class_exists(cls) if cls else True
|
||
est = cfg.get("extra_special_tokens")
|
||
problems = []
|
||
if cls and known is False:
|
||
problems.append(f"tokenizer_class '{cls}' not found in transformers")
|
||
if isinstance(est, list):
|
||
problems.append(f"extra_special_tokens is a list (n={len(est)})")
|
||
if cfg.get("from_slow") and "tokenizer.json" in set(files):
|
||
problems.append("from_slow=true while tokenizer.json is present")
|
||
if not problems:
|
||
return "" # 干净模型:什么都不做,行为与原始镜像完全一致
|
||
|
||
log("detected: " + "; ".join(problems))
|
||
|
||
candidates = build_candidates(cfg, files)
|
||
if not candidates:
|
||
log("no applicable fix (no tokenizer data files, no copyable remote code), skip")
|
||
return ""
|
||
|
||
build_dir = out_dir + ".build"
|
||
shutil.rmtree(build_dir, ignore_errors=True)
|
||
try:
|
||
copy_payload(model_dir, build_dir, files)
|
||
for name, cand, needs_remote in candidates:
|
||
write_config(os.path.join(build_dir, "tokenizer_config.json"), cand)
|
||
if not verify:
|
||
log(f"candidate [{name}] not verified (FIX_TOKENIZER_VERIFY=0)")
|
||
break
|
||
reasons = []
|
||
for trust in ([True, None] if needs_remote else [None, True]):
|
||
err = try_load(build_dir, trust)
|
||
if err is None:
|
||
log(f"candidate [{name}] loaded OK (trust_remote_code={trust})")
|
||
break
|
||
reasons.append(f"trust={trust}: {err}")
|
||
else:
|
||
log(f"candidate [{name}] failed: {reasons[0]}")
|
||
continue
|
||
break
|
||
else:
|
||
log("all candidates failed to load, skip fix (starting vLLM as-is)")
|
||
shutil.rmtree(build_dir, ignore_errors=True)
|
||
return ""
|
||
|
||
if name.startswith("rewrite-class"):
|
||
log("warning: fell back to class rewrite, tokenization may differ from the repo's own tokenizer")
|
||
if dry_run:
|
||
log(f"[dry-run] would use candidate [{name}] at {build_dir}")
|
||
return build_dir
|
||
|
||
shutil.rmtree(out_dir, ignore_errors=True)
|
||
os.replace(build_dir, out_dir)
|
||
return out_dir
|
||
except Exception: # noqa: BLE001 - 补丁自身绝不能把容器搞挂
|
||
log("fix failed, starting as-is:\n" + traceback.format_exc())
|
||
shutil.rmtree(build_dir, ignore_errors=True)
|
||
return ""
|
||
|
||
|
||
def main(argv: Optional[List[str]] = None) -> int:
|
||
ap = argparse.ArgumentParser(
|
||
description="detect and fix tokenizer_config.json incompatibilities with vLLM 0.9.1 (prints the fixed dir path on stdout)"
|
||
)
|
||
ap.add_argument("model_dir", nargs="?", default=os.environ.get("MODEL_DIR", "/model"))
|
||
ap.add_argument("--out-dir", default=os.environ.get("FIX_TOKENIZER_OUT_DIR", DEFAULT_OUT_DIR))
|
||
ap.add_argument("--no-verify", action="store_true",
|
||
help="skip the trial load (same as FIX_TOKENIZER_VERIFY=0)")
|
||
ap.add_argument("--dry-run", action="store_true",
|
||
help="only report which candidate would be used, write nothing (debug)")
|
||
ap.add_argument("--max-copy-mb", type=int,
|
||
default=int(os.environ.get("FIX_TOKENIZER_MAX_COPY_MB", "256")))
|
||
args = ap.parse_args(argv)
|
||
|
||
verify_env = os.environ.get("FIX_TOKENIZER_VERIFY", "1").strip().lower()
|
||
verify = (not args.no_verify) and verify_env not in ("0", "false", "no", "off")
|
||
|
||
try:
|
||
out = fix(
|
||
args.model_dir,
|
||
args.out_dir,
|
||
verify=verify,
|
||
max_bytes=max(1, args.max_copy_mb) * 1024 * 1024,
|
||
dry_run=args.dry_run,
|
||
)
|
||
except Exception: # noqa: BLE001
|
||
log("unhandled error, starting as-is:\n" + traceback.format_exc())
|
||
out = ""
|
||
|
||
if out:
|
||
print(out) # stdout 只留这一行,wrapper 用它
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|