37 lines
1.1 KiB
Python
37 lines
1.1 KiB
Python
|
import os
|
|||
|
|
import json
|
|||
|
|
|
|||
|
|
|
|||
|
|
def detect(model_dir):
|
|||
|
|
"""判定 tokenizer 类型：fast / sentencepiece / bpe / unknown
|
|||
|
|
|
|||
|
|
与社区已上线引擎 EngineX-Sunrise/enginex-S2-vllm-fix-tokenizer 的
|
|||
|
|
detect_tokenizer.py 保持一致（同一判定口径，便于两套引擎交叉验证）。
|
|||
|
|
"""
|
|||
|
|
cfg_path = os.path.join(model_dir, "tokenizer_config.json")
|
|||
|
|
cls = ""
|
|||
|
|
if os.path.exists(cfg_path):
|
|||
|
|
try:
|
|||
|
|
with open(cfg_path, encoding="utf-8") as f:
|
|||
|
|
cls = (json.load(f) or {}).get("tokenizer_class", "") or ""
|
|||
|
|
except Exception: # noqa: BLE001
|
|||
|
|
cls = ""
|
|||
|
|
try:
|
|||
|
|
files = set(os.listdir(model_dir))
|
|||
|
|
except Exception: # noqa: BLE001
|
|||
|
|
files = set()
|
|||
|
|
|
|||
|
|
if "tokenizer.json" in files:
|
|||
|
|
return "fast", cls
|
|||
|
|
if "tokenizer.model" in files:
|
|||
|
|
return "sentencepiece", cls
|
|||
|
|
if "vocab.json" in files and "merges.txt" in files:
|
|||
|
|
return "bpe", cls
|
|||
|
|
return "unknown", cls
|
|||
|
|
|
|||
|
|
|
|||
|
|
if __name__ == "__main__":
|
|||
|
|
import sys
|
|||
|
|
t, c = detect(sys.argv[1] if len(sys.argv) > 1 else "/model")
|
|||
|
|
print(t, c)
|