add fix extra_special_tokens: list → dict

Signed-off-by: Sun Ruoxi <sunruoxi@4paradigm.com>

fix_tokenizer.py

@@ -56,6 +56,12 @@ bad_classes = [
 if orig_cls in bad_classes:
     print(f"[fix] override bad tokenizer_class: {orig_cls} → {cfg['tokenizer_class']}")
 
+# 修复 extra_special_tokens: list → dict 格式
+if "extra_special_tokens" in cfg and isinstance(cfg["extra_special_tokens"], list):
+    orig_list = cfg["extra_special_tokens"]
+    cfg["extra_special_tokens"] = {token: token for token in orig_list}
+    print(f"[fix] converted extra_special_tokens from list ({len(orig_list)} items) to dict format")
+
 # 写回
 with open(cfg_path, "w") as f:
     json.dump(cfg, f)