diff --git a/fix_tokenizer.py b/fix_tokenizer.py index 9556ea6..67563c2 100644 --- a/fix_tokenizer.py +++ b/fix_tokenizer.py @@ -37,6 +37,8 @@ else: # ===== 自动修复策略 ===== if typ == "fast": cfg["tokenizer_class"] = "PreTrainedTokenizerFast" + cfg["from_slow"] = False + cfg.pop("backend", None) elif typ == "sentencepiece": cfg["tokenizer_class"] = "LlamaTokenizer" @@ -55,6 +57,7 @@ bad_classes = [ if orig_cls in bad_classes: print(f"[fix] override bad tokenizer_class: {orig_cls} → {cfg['tokenizer_class']}") + print(f"[fix] override from_slow: {cfg['from_slow']}") # 修复 extra_special_tokens: list → dict 格式 if "extra_special_tokens" in cfg and isinstance(cfg["extra_special_tokens"], list):