From 772e104e50c7948ac5de8f03122766a7672177ef Mon Sep 17 00:00:00 2001
From: huni <chenweize1202@gmail.com>
Date: Mon, 24 Aug 2026 18:07:11 +0800
Subject: [PATCH] add detect_tokenizer.py

---
 detect_tokenizer.py | 25 +++++++++++++++++++++++++
 1 file changed, 25 insertions(+)
 create mode 100644 detect_tokenizer.py

diff --git a/detect_tokenizer.py b/detect_tokenizer.py
new file mode 100644
index 0000000..c0e7b3e
--- /dev/null
+++ b/detect_tokenizer.py
@@ -0,0 +1,25 @@
+import os
+import json
+
+def detect(model_dir):
+    cfg_path = os.path.join(model_dir, "tokenizer_config.json")
+
+    if os.path.exists(cfg_path):
+        with open(cfg_path) as f:
+            cfg = json.load(f)
+        cls = cfg.get("tokenizer_class", "")
+    else:
+        cls = ""
+
+    files = os.listdir(model_dir)
+
+    if "tokenizer.json" in files:
+        return "fast", cls
+
+    if "tokenizer.model" in files:
+        return "sentencepiece", cls
+
+    if "vocab.json" in files and "merges.txt" in files:
+        return "bpe", cls
+
+    return "unknown", cls