diff --git a/fix_tokenizer.py b/fix_tokenizer.py index 9556ea6..8e012a3 100644 --- a/fix_tokenizer.py +++ b/fix_tokenizer.py @@ -37,6 +37,8 @@ else: # ===== 自动修复策略 ===== if typ == "fast": cfg["tokenizer_class"] = "PreTrainedTokenizerFast" + cfg["from_slow"] = False + cfg.pop("backend", None) elif typ == "sentencepiece": cfg["tokenizer_class"] = "LlamaTokenizer"