sunruoxi/enginex-xc-llm-kunlunxin-fix-tokenizer
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

更新 fix_tokenizer.py

Browse Source
This commit is contained in:
sunruoxi
2026-07-21 16:55:50 +08:00
parent b324b7140a
commit 5d34d33e0d
1 changed files with 2 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
fix_tokenizer.py
View File

@@ -37,6 +37,8 @@ else:
# ===== 自动修复策略 ===== # ===== 自动修复策略 =====
if typ == "fast": if typ == "fast":
cfg["tokenizer_class"] = "PreTrainedTokenizerFast" cfg["tokenizer_class"] = "PreTrainedTokenizerFast"
cfg["from_slow"] = False
cfg.pop("backend", None)
elif typ == "sentencepiece": elif typ == "sentencepiece":
cfg["tokenizer_class"] = "LlamaTokenizer" cfg["tokenizer_class"] = "LlamaTokenizer"