Files

356 lines
15 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
"""检测并修复 tokenizer_config.json 里与 vLLM 0.9.1(transformers 4.x)不兼容的写法。
面向 MetaX C500:容器启动时由 `vllm_wrapper.sh` 调用,
把模型目录里的 tokenizer 相关文件拷到临时目录、改好配置后用 `--tokenizer` 指过去,
**原始模型目录不做任何修改**。
## 修什么
| # | 缺陷 | 现场报错(vLLM 0.9.1 / transformers 4.5x) |
|---|---|---|
| 1 | `tokenizer_class` 不是 transformers 合法类名(典型 `TokenizersBackend`,新版本 transformers 写出) | `ValueError: Tokenizer class X does not exist or is not currently imported.` |
| 2 | `extra_special_tokens` 写成 list | `AttributeError: 'list' object has no attribute 'keys'` |
| 3 | 有 `tokenizer.json` 但 `from_slow: true` | `ValueError: Cannot instantiate this tokenizer from a slow version…`(transformers 会绕开 tokenizer.json 走 slow 路径,而 `PreTrainedTokenizerFast` 没有 slow 类) |
| 4 | 带 `auto_map` 的仓库自带 tokenizer 代码(`QWenTokenizer` / `MuPTTokenizer` / `InternLM2Tokenizer` …) | 改写 `tokenizer_class` 后 `auto_map` 仍生效,但 `/tmp/fixed_tokenizer` 里没有 `tokenization_*.py`:`OSError: … does not appear to have a file named tokenization_qwen.py` |
第 4 类**不能靠改写类名解决**:这些 tokenizer 是仓库自带 Python 代码实现的(还常带私有词表,
如 Qwen1 的 `qwen.tiktoken`),换成 `GPT2TokenizerFast` 之类的通用类会**改变分词结果**
(实测 MuPT 对 `<|endoftext|>` 的编码就不一样)。所以正确做法是把代码和词表一起拷过去、
保留 `auto_map`,让仓库自己的 tokenizer 跑起来。
## 怎么保证不把「能起」改成「起不来」
候选配置按「改动最小 → 逐级兜底」排队,**每个候选都真的用 `AutoTokenizer.from_pretrained(dst)`
试加载一次**(走 encode 一遍),第一个成功的才落到输出目录;全都失败就什么都不输出,
wrapper 于是按原样启动 vLLM —— 等价于没有这个补丁,绝不会更差。
候选顺序:
1. **最小改动**:只修 `extra_special_tokens` / `from_slow`,`tokenizer_class` 和 `auto_map` 原样保留
(类名合法、或 `auto_map` 指向的 `.py` 已随文件拷过去时可选);
2. **改写类名**:按目录里的数据文件挑 `PreTrainedTokenizerFast` / `LlamaTokenizer` / `GPT2TokenizerFast`,
同时去掉 `auto_map`(否则动态加载会去找已不存在的 `.py`)。
## 输出契约(wrapper 依赖)
* stdout:**只有一行**——可用的修复目录(没有就什么都不输出,退出码 0);
* stderr:`[fix_tokenizer]` 开头的诊断日志;
* 任何异常都吞掉并返回 0(不让补丁本身把容器搞挂)。
环境变量:
* `MODEL_DIR`:模型目录(也可用第一个位置参数);
* `FIX_TOKENIZER_OUT_DIR`:输出目录,默认 `/tmp/fixed_tokenizer`;
* `FIX_TOKENIZER_VERIFY=0`:跳过试加载(不建议;离线调试用);
* `FIX_TOKENIZER_MAX_COPY_MB`:单个文件的拷贝上限,默认 256(防止把权重拷进去)。
适配 MetaX C500 / vLLM 0.9.1 环境。
"""
from __future__ import annotations
import argparse
import json
import os
import shutil
import sys
import traceback
from typing import Dict, Iterable, List, Optional, Tuple
DEFAULT_OUT_DIR = "/tmp/fixed_tokenizer"
# 权重容器:一律不拷(tokenizer 目录用不到,模型动辄几十 GB)
SKIP_SUFFIXES = (
".safetensors", ".bin", ".pt", ".pth", ".ckpt", ".gguf", ".onnx",
".msgpack", ".h5", ".npz", ".npy", ".pkl", ".sqlite", ".db",
)
# transformers 5.x 才认识的键,4.x 里没有意义;留着会被原样透传给 tokenizer 构造函数
V5_ONLY_KEYS = ("from_slow", "backend")
def log(msg: str) -> None:
print(f"[fix_tokenizer] {msg}", file=sys.stderr, flush=True)
# ── transformers 探测 ──────────────────────────────────────────────
def _transformers():
try:
import transformers # noqa: F401
except Exception: # pragma: no cover - 镜像里一定有
return None
return transformers
def class_exists(name: str) -> Optional[bool]:
"""类在 transformers 里是否存在。None = 判断不了(没装 transformers)。"""
tf = _transformers()
if tf is None or not name:
return None
return getattr(tf, name, None) is not None
# ── 文件 ──────────────────────────────────────────────────────────
def payload_files(model_dir: str, max_bytes: int) -> List[str]:
"""模型目录里值得拷进修复目录的文件(tokenizer 数据 + 仓库自带代码)。"""
out: List[str] = []
try:
names = sorted(os.listdir(model_dir))
except OSError:
return out
for name in names:
if name.startswith("."):
continue
if name.endswith(SKIP_SUFFIXES):
continue
src = os.path.join(model_dir, name)
if not os.path.isfile(src):
continue
try:
if os.path.getsize(src) > max_bytes:
continue
except OSError:
continue
out.append(name)
return out
def copy_payload(model_dir: str, dst: str, names: Iterable[str]) -> None:
os.makedirs(dst, exist_ok=True)
for name in names:
shutil.copy2(os.path.join(model_dir, name), os.path.join(dst, name))
def read_config(path: str) -> Optional[Dict]:
try:
with open(path, "r", encoding="utf-8") as fh:
data = json.load(fh)
except (OSError, ValueError):
return None
return data if isinstance(data, dict) else None
def write_config(path: str, cfg: Dict) -> None:
with open(path, "w", encoding="utf-8") as fh:
json.dump(cfg, fh, indent=2, ensure_ascii=False)
# ── 配置分析与候选 ────────────────────────────────────────────────
def auto_map_modules(cfg: Dict) -> List[str]:
"""`auto_map` 里 AutoTokenizer 指向的模块文件名(如 tokenization_qwen.py)。"""
am = cfg.get("auto_map")
entry = None
if isinstance(am, dict):
entry = am.get("AutoTokenizer")
elif isinstance(am, (list, tuple)): # 老格式:["tokenization_x.X", null]
entry = am
if not entry:
return []
refs = entry if isinstance(entry, (list, tuple)) else [entry]
mods: List[str] = []
for ref in refs:
if not isinstance(ref, str) or not ref:
continue
mod = ref.split(".")[0].strip()
if mod and f"{mod}.py" not in mods:
mods.append(f"{mod}.py")
return mods
def choose_class(files: Iterable[str]) -> Optional[str]:
"""按目录里实际存在的数据文件挑一个能实例化的 transformers 类。"""
names = set(files)
if "tokenizer.json" in names:
return "PreTrainedTokenizerFast"
if "tokenizer.model" in names:
return "LlamaTokenizer"
if "vocab.json" in names and "merges.txt" in names:
return "GPT2TokenizerFast"
return None
def apply_common_fixes(cfg: Dict, files: Iterable[str], force_fast: bool) -> Tuple[Dict, List[str]]:
"""两类「无条件该修」的项:extra_special_tokens list→dict、会误伤 tokenizer.json 的 from_slow。"""
new = json.loads(json.dumps(cfg)) # 深拷贝,不改入参
fixes: List[str] = []
names = set(files)
est = new.get("extra_special_tokens")
if isinstance(est, list):
new["extra_special_tokens"] = {str(t): str(t) for t in est}
fixes.append(f"extra_special_tokens: list({len(est)}) -> dict")
if new.get("from_slow") and (force_fast or "tokenizer.json" in names):
# from_slow=true 时 transformers 会丢弃 tokenizer.json 走 slow 转换;
# 目标类若是通用 fast 类(没有 slow_tokenizer_class)就会直接抛 slow-version 错误。
new.pop("from_slow", None)
fixes.append("from_slow: true dropped (use tokenizer.json)")
return new, fixes
def build_candidates(cfg: Dict, files: List[str]) -> List[Tuple[str, Dict, bool]]:
"""返回 [(说明, 候选配置, 是否需要 trust_remote_code)],按「改动最小」排序。"""
names = set(files)
cls = cfg.get("tokenizer_class") or ""
known = class_exists(cls) if cls else False
mods = auto_map_modules(cfg)
remote_ready = bool(mods) and all(m in names for m in mods)
out: List[Tuple[str, Dict, bool]] = []
# 候选 1:尽量不动,只做必改项;auto_map 的代码已经拷过去,让仓库自己的 tokenizer 跑
if known or remote_ready:
cand, fixes = apply_common_fixes(cfg, names, force_fast=False)
note = "keep-class"
if remote_ready:
note += "+auto_map"
out.append((note, cand, remote_ready))
# 候选 2:改写类名 + 去掉 auto_map(最后兜底,可能改变分词口径,日志里会警告)
replacement = choose_class(names)
if replacement and replacement != cls:
cand, fixes = apply_common_fixes(cfg, names, force_fast=True)
cand["tokenizer_class"] = replacement
cand.pop("auto_map", None)
if fixes:
log("rewrite-class fixes: " + "; ".join(fixes))
out.append((f"rewrite-class->{replacement}", cand, False))
return out
# ── 试加载 ────────────────────────────────────────────────────────
def try_load(directory: str, trust_remote_code: Optional[bool]) -> Optional[str]:
"""真的加载一次 tokenizer 并 encode,成功返回 None,失败返回原因。"""
try:
from transformers import AutoTokenizer
except Exception as exc: # pragma: no cover - 镜像里一定有
return f"transformers unavailable: {exc}"
try:
tok = AutoTokenizer.from_pretrained(directory, trust_remote_code=trust_remote_code)
tok.encode("hello world")
return None
except Exception as exc:
first = str(exc).splitlines()[0] if str(exc) else type(exc).__name__
return f"{type(exc).__name__}: {first[:180]}"
# ── 主流程 ────────────────────────────────────────────────────────
def fix(model_dir: str, out_dir: str, verify: bool, max_bytes: int, dry_run: bool) -> str:
cfg_path = os.path.join(model_dir, "tokenizer_config.json")
cfg = read_config(cfg_path)
if cfg is None:
log(f"no readable tokenizer_config.json at {cfg_path}, skip")
return ""
files = payload_files(model_dir, max_bytes)
if not files:
log("no copyable tokenizer files in model dir, skip")
return ""
cls = cfg.get("tokenizer_class") or ""
known = class_exists(cls) if cls else True
est = cfg.get("extra_special_tokens")
problems = []
if cls and known is False:
problems.append(f"tokenizer_class '{cls}' not found in transformers")
if isinstance(est, list):
problems.append(f"extra_special_tokens is a list (n={len(est)})")
if cfg.get("from_slow") and "tokenizer.json" in set(files):
problems.append("from_slow=true while tokenizer.json is present")
if not problems:
return "" # 干净模型:什么都不做,行为与原始镜像完全一致
log("detected: " + "; ".join(problems))
candidates = build_candidates(cfg, files)
if not candidates:
log("no applicable fix (no tokenizer data files, no copyable remote code), skip")
return ""
build_dir = out_dir + ".build"
shutil.rmtree(build_dir, ignore_errors=True)
try:
copy_payload(model_dir, build_dir, files)
for name, cand, needs_remote in candidates:
write_config(os.path.join(build_dir, "tokenizer_config.json"), cand)
if not verify:
log(f"candidate [{name}] not verified (FIX_TOKENIZER_VERIFY=0)")
break
reasons = []
for trust in ([True, None] if needs_remote else [None, True]):
err = try_load(build_dir, trust)
if err is None:
log(f"candidate [{name}] loaded OK (trust_remote_code={trust})")
break
reasons.append(f"trust={trust}: {err}")
else:
log(f"candidate [{name}] failed: {reasons[0]}")
continue
break
else:
log("all candidates failed to load, skip fix (starting vLLM as-is)")
shutil.rmtree(build_dir, ignore_errors=True)
return ""
if name.startswith("rewrite-class"):
log("warning: fell back to class rewrite, tokenization may differ from the repo's own tokenizer")
if dry_run:
log(f"[dry-run] would use candidate [{name}] at {build_dir}")
return build_dir
shutil.rmtree(out_dir, ignore_errors=True)
os.replace(build_dir, out_dir)
return out_dir
except Exception: # noqa: BLE001 - 补丁自身绝不能把容器搞挂
log("fix failed, starting as-is:\n" + traceback.format_exc())
shutil.rmtree(build_dir, ignore_errors=True)
return ""
def main(argv: Optional[List[str]] = None) -> int:
ap = argparse.ArgumentParser(
description="detect and fix tokenizer_config.json incompatibilities with vLLM 0.9.1 (prints the fixed dir path on stdout)"
)
ap.add_argument("model_dir", nargs="?", default=os.environ.get("MODEL_DIR", "/model"))
ap.add_argument("--out-dir", default=os.environ.get("FIX_TOKENIZER_OUT_DIR", DEFAULT_OUT_DIR))
ap.add_argument("--no-verify", action="store_true",
help="skip the trial load (same as FIX_TOKENIZER_VERIFY=0)")
ap.add_argument("--dry-run", action="store_true",
help="only report which candidate would be used, write nothing (debug)")
ap.add_argument("--max-copy-mb", type=int,
default=int(os.environ.get("FIX_TOKENIZER_MAX_COPY_MB", "256")))
args = ap.parse_args(argv)
verify_env = os.environ.get("FIX_TOKENIZER_VERIFY", "1").strip().lower()
verify = (not args.no_verify) and verify_env not in ("0", "false", "no", "off")
try:
out = fix(
args.model_dir,
args.out_dir,
verify=verify,
max_bytes=max(1, args.max_copy_mb) * 1024 * 1024,
dry_run=args.dry_run,
)
except Exception: # noqa: BLE001
log("unhandled error, starting as-is:\n" + traceback.format_exc())
out = ""
if out:
print(out) # stdout 只留这一行,wrapper 用它
return 0
if __name__ == "__main__":
main()