更新 fix_tokenizer.py

2026-07-21 16:55:50 +08:00
parent b324b7140a
commit 5d34d33e0d
1 changed files with 2 additions and 0 deletions
--- a/fix_tokenizer.py
+++ b/fix_tokenizer.py
@@ -37,6 +37,8 @@ else:
 # ===== 自动修复策略 =====
 if typ == "fast":
    cfg["tokenizer_class"] = "PreTrainedTokenizerFast"
+    cfg["from_slow"] = False
+    cfg.pop("backend", None)

 elif typ == "sentencepiece":
    cfg["tokenizer_class"] = "LlamaTokenizer"