forked from facebook/MobileLLM-R1-360M
19 lines
560 B
Python
19 lines
560 B
Python
|
#!/usr/bin/env python3
|
||
|
|
# 检测 tokenizer 是否需要修复
|
||
|
|
import json
|
||
|
|
import os
|
||
|
|
|
||
|
|
MODEL_DIR = os.environ.get("MODEL_DIR", "/model")
|
||
|
|
cfg_path = os.path.join(MODEL_DIR, "tokenizer_config.json")
|
||
|
|
if not os.path.exists(cfg_path):
|
||
|
|
print("no tokenizer_config.json")
|
||
|
|
raise SystemExit(0)
|
||
|
|
with open(cfg_path, "r", encoding="utf-8") as f:
|
||
|
|
cfg = json.load(f)
|
||
|
|
if cfg.get("tokenizer_class") in {"TokenizersBackend", "TiktokenTokenizer"}:
|
||
|
|
print("need_fix")
|
||
|
|
elif isinstance(cfg.get("extra_special_tokens"), list):
|
||
|
|
print("need_fix")
|
||
|
|
else:
|
||
|
|
print("ok")
|