From b71021d7e7d3437aadcffcdcc1bd446c7de0bccd Mon Sep 17 00:00:00 2001
From: i-peixingyu <i-peixingyu@4paradigm.com>
Date: Tue, 7 Jul 2026 11:25:28 +0800
Subject: [PATCH] =?UTF-8?q?=E5=88=A0=E9=99=A4=20NV=20A100=20Patched=20?=
 =?UTF-8?q?=E9=95=9C=E5=83=8F=E5=90=88=E5=B9=B6/fix=5Ftokenizer.py?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 NV A100 Patched 镜像合并/fix_tokenizer.py | 64 -----------------------
 1 file changed, 64 deletions(-)
 delete mode 100644 NV A100 Patched 镜像合并/fix_tokenizer.py

diff --git a/NV A100 Patched 镜像合并/fix_tokenizer.py b/NV A100 Patched 镜像合并/fix_tokenizer.py
deleted file mode 100644
index 2981d52..0000000
--- a/NV A100 Patched 镜像合并/fix_tokenizer.py	
+++ /dev/null
@@ -1,64 +0,0 @@
-import os 
-import shutil
-import json
-import transformers
-import inspect
-from detect_tokenizer import detect
-
-MODEL_DIR = os.environ.get("MODEL_DIR", "/model")
-OUT_DIR = os.environ.get("FIX_TOKENIZER_DIR", "/tmp/fixed_tokenizer")
-
-os.makedirs(OUT_DIR, exist_ok=True)
-
-def copy_if_exists(name):
-    src = os.path.join(MODEL_DIR, name)
-    if os.path.exists(src):
-        shutil.copy(src, OUT_DIR)
-
-for f in [
-    "tokenizer.json",
-    "tokenizer_config.json",
-    "special_tokens_map.json",
-    "vocab.json",
-    "merges.txt",
-    "tokenizer.model",
-]:
-    copy_if_exists(f)
-
-typ, orig_cls = detect(MODEL_DIR)
-
-cfg_path = os.path.join(OUT_DIR, "tokenizer_config.json")
-
-if os.path.exists(cfg_path):
-    with open(cfg_path) as f:
-        cfg = json.load(f)
-else:
-    cfg = {}
-
-VALID_CLASSES = {
-    name for name, obj in inspect.getmembers(transformers)
-    if inspect.isclass(obj) and "Tokenizer" in name
-}
-
-BAD_CLASSES = {"TokenizersBackend", "TiktokenTokenizer"}
-
-FALLBACK = {
-    "fast": "PreTrainedTokenizerFast",
-    "sentencepiece": "LlamaTokenizer",
-    "bpe": "GPT2TokenizerFast",
-}
-
-if orig_cls and orig_cls in VALID_CLASSES and orig_cls not in BAD_CLASSES:
-    print(f"[fix_tokenizer] tokenizer_class '{orig_cls}' is valid, skip override")
-else:
-    fallback = FALLBACK.get(typ, "PreTrainedTokenizerFast")
-    if orig_cls:
-        print(f"[fix] override bad tokenizer_class: {orig_cls} → {fallback}")
-    else:
-        print(f"[fix] tokenizer_class missing, set to: {fallback}")
-    cfg["tokenizer_class"] = fallback
-
-with open(cfg_path, "w") as f:
-    json.dump(cfg, f)
-
-print(f"[fix_tokenizer] done → {OUT_DIR}")
\ No newline at end of file