diff --git a/detect_tokenizer.py b/detect_tokenizer.py new file mode 100644 index 0000000..ede8122 --- /dev/null +++ b/detect_tokenizer.py @@ -0,0 +1,36 @@ +import os +import json + + +def detect(model_dir): + """判定 tokenizer 类型:fast / sentencepiece / bpe / unknown + + 与社区已上线引擎 EngineX-Sunrise/enginex-S2-vllm-fix-tokenizer 的 + detect_tokenizer.py 保持一致(同一判定口径,便于两套引擎交叉验证)。 + """ + cfg_path = os.path.join(model_dir, "tokenizer_config.json") + cls = "" + if os.path.exists(cfg_path): + try: + with open(cfg_path, encoding="utf-8") as f: + cls = (json.load(f) or {}).get("tokenizer_class", "") or "" + except Exception: # noqa: BLE001 + cls = "" + try: + files = set(os.listdir(model_dir)) + except Exception: # noqa: BLE001 + files = set() + + if "tokenizer.json" in files: + return "fast", cls + if "tokenizer.model" in files: + return "sentencepiece", cls + if "vocab.json" in files and "merges.txt" in files: + return "bpe", cls + return "unknown", cls + + +if __name__ == "__main__": + import sys + t, c = detect(sys.argv[1] if len(sys.argv) > 1 else "/model") + print(t, c)