From 5d34d33e0dbab1cdb56a962874da7cdabe4e5024 Mon Sep 17 00:00:00 2001
From: sunruoxi <sunruoxi@4paradigm.com>
Date: Tue, 21 Jul 2026 16:55:50 +0800
Subject: [PATCH] =?UTF-8?q?=E6=9B=B4=E6=96=B0=20fix=5Ftokenizer.py?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 fix_tokenizer.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/fix_tokenizer.py b/fix_tokenizer.py
index 9556ea6..8e012a3 100644
--- a/fix_tokenizer.py
+++ b/fix_tokenizer.py
@@ -37,6 +37,8 @@ else:
 # ===== 自动修复策略 =====
 if typ == "fast":
     cfg["tokenizer_class"] = "PreTrainedTokenizerFast"
+    cfg["from_slow"] = False
+    cfg.pop("backend", None)
 
 elif typ == "sentencepiece":
     cfg["tokenizer_class"] = "LlamaTokenizer"