EngineX-Sunrise/enginex-S2-vllm-fix-tokenizer
5
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

更新 fix_tokenizer.py
All checks were successful
Docker Build and Push / docker (push) Successful in 22m39s
Details

Browse Source
This commit is contained in:
sunruoxi
2026-07-22 11:37:03 +08:00
parent 3725866cc5
commit c71f8b10fd
1 changed files with 3 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
fix_tokenizer.py
View File

@@ -37,6 +37,8 @@ else:
# ===== 自动修复策略 ===== # ===== 自动修复策略 =====
if typ == "fast": if typ == "fast":
cfg["tokenizer_class"] = "PreTrainedTokenizerFast" cfg["tokenizer_class"] = "PreTrainedTokenizerFast"
cfg["from_slow"] = False
cfg.pop("backend", None)
elif typ == "sentencepiece": elif typ == "sentencepiece":
cfg["tokenizer_class"] = "LlamaTokenizer" cfg["tokenizer_class"] = "LlamaTokenizer"
@@ -55,6 +57,7 @@ bad_classes = [
if orig_cls in bad_classes: if orig_cls in bad_classes:
print(f"[fix] override bad tokenizer_class: {orig_cls} → {cfg['tokenizer_class']}") print(f"[fix] override bad tokenizer_class: {orig_cls} → {cfg['tokenizer_class']}")
print(f"[fix] override from_slow: {cfg['from_slow']}")
# 修复 extra_special_tokens: list → dict 格式 # 修复 extra_special_tokens: list → dict 格式
if "extra_special_tokens" in cfg and isinstance(cfg["extra_special_tokens"], list): if "extra_special_tokens" in cfg and isinstance(cfg["extra_special_tokens"], list):