更新 fix_tokenizer.py

2026-07-21 18:46:53 +08:00
parent 8c50b2d368
commit 68c7b9c52f
1 changed files with 2 additions and 0 deletions
--- a/fix_tokenizer.py
+++ b/fix_tokenizer.py
@@ -37,6 +37,8 @@ else:
 # ===== 自动修复策略 =====
 if typ == "fast":
    cfg["tokenizer_class"] = "PreTrainedTokenizerFast"
+    cfg["from_slow"] = False
+    cfg.pop("backend", None)

 elif typ == "sentencepiece":
    cfg["tokenizer_class"] = "LlamaTokenizer"