#!/usr/bin/env python3
# 检测 tokenizer 是否需要修复
import json
import os

MODEL_DIR = os.environ.get("MODEL_DIR", "/model")
cfg_path = os.path.join(MODEL_DIR, "tokenizer_config.json")
if not os.path.exists(cfg_path):
    print("no tokenizer_config.json")
    raise SystemExit(0)
with open(cfg_path, "r", encoding="utf-8") as f:
    cfg = json.load(f)
if cfg.get("tokenizer_class") in {"TokenizersBackend", "TiktokenTokenizer"}:
    print("need_fix")
elif isinstance(cfg.get("extra_special_tokens"), list):
    print("need_fix")
else:
    print("ok")