From c71f8b10fd9b314bf76c793aa5845aae51fd7f60 Mon Sep 17 00:00:00 2001
From: sunruoxi <sunruoxi@4paradigm.com>
Date: Wed, 22 Jul 2026 11:37:03 +0800
Subject: [PATCH] =?UTF-8?q?=E6=9B=B4=E6=96=B0=20fix=5Ftokenizer.py?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 fix_tokenizer.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/fix_tokenizer.py b/fix_tokenizer.py
index 9556ea6..67563c2 100644
--- a/fix_tokenizer.py
+++ b/fix_tokenizer.py
@@ -37,6 +37,8 @@ else:
 # ===== 自动修复策略 =====
 if typ == "fast":
     cfg["tokenizer_class"] = "PreTrainedTokenizerFast"
+    cfg["from_slow"] = False
+    cfg.pop("backend", None)
 
 elif typ == "sentencepiece":
     cfg["tokenizer_class"] = "LlamaTokenizer"
@@ -55,6 +57,7 @@ bad_classes = [
 
 if orig_cls in bad_classes:
     print(f"[fix] override bad tokenizer_class: {orig_cls} → {cfg['tokenizer_class']}")
+    print(f"[fix] override from_slow: {cfg['from_slow']}")
 
 # 修复 extra_special_tokens: list → dict 格式
 if "extra_special_tokens" in cfg and isinstance(cfg["extra_special_tokens"], list):