enginex-mlu370-compat: detect_tokenizer.py

detect_tokenizer.py

@@ -0,0 +1,36 @@
+import os
+import json
+
+
+def detect(model_dir):
+    """判定 tokenizer 类型：fast / sentencepiece / bpe / unknown
+
+    与社区已上线引擎 EngineX-Sunrise/enginex-S2-vllm-fix-tokenizer 的
+    detect_tokenizer.py 保持一致（同一判定口径，便于两套引擎交叉验证）。
+    """
+    cfg_path = os.path.join(model_dir, "tokenizer_config.json")
+    cls = ""
+    if os.path.exists(cfg_path):
+        try:
+            with open(cfg_path, encoding="utf-8") as f:
+                cls = (json.load(f) or {}).get("tokenizer_class", "") or ""
+        except Exception:  # noqa: BLE001
+            cls = ""
+    try:
+        files = set(os.listdir(model_dir))
+    except Exception:  # noqa: BLE001
+        files = set()
+
+    if "tokenizer.json" in files:
+        return "fast", cls
+    if "tokenizer.model" in files:
+        return "sentencepiece", cls
+    if "vocab.json" in files and "merges.txt" in files:
+        return "bpe", cls
+    return "unknown", cls
+
+
+if __name__ == "__main__":
+    import sys
+    t, c = detect(sys.argv[1] if len(sys.argv) > 1 else "/model")
+    print(t, c)