37 lines
1.1 KiB
Python
37 lines
1.1 KiB
Python
import os
|
||
import json
|
||
|
||
|
||
def detect(model_dir):
|
||
"""判定 tokenizer 类型：fast / sentencepiece / bpe / unknown
|
||
|
||
与社区已上线引擎 EngineX-Sunrise/enginex-S2-vllm-fix-tokenizer 的
|
||
detect_tokenizer.py 保持一致（同一判定口径，便于两套引擎交叉验证）。
|
||
"""
|
||
cfg_path = os.path.join(model_dir, "tokenizer_config.json")
|
||
cls = ""
|
||
if os.path.exists(cfg_path):
|
||
try:
|
||
with open(cfg_path, encoding="utf-8") as f:
|
||
cls = (json.load(f) or {}).get("tokenizer_class", "") or ""
|
||
except Exception: # noqa: BLE001
|
||
cls = ""
|
||
try:
|
||
files = set(os.listdir(model_dir))
|
||
except Exception: # noqa: BLE001
|
||
files = set()
|
||
|
||
if "tokenizer.json" in files:
|
||
return "fast", cls
|
||
if "tokenizer.model" in files:
|
||
return "sentencepiece", cls
|
||
if "vocab.json" in files and "merges.txt" in files:
|
||
return "bpe", cls
|
||
return "unknown", cls
|
||
|
||
|
||
if __name__ == "__main__":
|
||
import sys
|
||
t, c = detect(sys.argv[1] if len(sys.argv) > 1 else "/model")
|
||
print(t, c)
|