add fix extra_special_tokens: list → dict

Signed-off-by: Sun Ruoxi <sunruoxi@4paradigm.com>
2026-05-28 17:56:05 +08:00
parent 8c08eb035b
commit 3725866cc5
3 changed files with 33 additions and 0 deletions
--- a/entrypoint.sh
+++ b/entrypoint.sh
@@ -18,6 +18,10 @@ elif [ "$AUTO_FIX" = "auto" ]; then
        if grep -q "TokenizersBackend\|TiktokenTokenizer" "$MODEL_DIR/tokenizer_config.json"; then
            NEED_FIX=1
        fi
+        # 检测 extra_special_tokens 是否为 list 格式
+        if grep -q '"extra_special_tokens":\s*\[' "$MODEL_DIR/tokenizer_config.json"; then
+            NEED_FIX=1
+        fi
    fi
 fi