import os
import json


def detect(model_dir):
    """判定 tokenizer 类型：fast / sentencepiece / bpe / unknown

    与社区已上线引擎 EngineX-Sunrise/enginex-S2-vllm-fix-tokenizer 的
    detect_tokenizer.py 保持一致（同一判定口径，便于两套引擎交叉验证）。
    """
    cfg_path = os.path.join(model_dir, "tokenizer_config.json")
    cls = ""
    if os.path.exists(cfg_path):
        try:
            with open(cfg_path, encoding="utf-8") as f:
                cls = (json.load(f) or {}).get("tokenizer_class", "") or ""
        except Exception:  # noqa: BLE001
            cls = ""
    try:
        files = set(os.listdir(model_dir))
    except Exception:  # noqa: BLE001
        files = set()

    if "tokenizer.json" in files:
        return "fast", cls
    if "tokenizer.model" in files:
        return "sentencepiece", cls
    if "vocab.json" in files and "merges.txt" in files:
        return "bpe", cls
    return "unknown", cls


if __name__ == "__main__":
    import sys
    t, c = detect(sys.argv[1] if len(sys.argv) > 1 else "/model")
    print(t, c)