24 lines
1.1 KiB
Diff
24 lines
1.1 KiB
Diff
|
--- convert_hf_to_gguf.py.original 2025-10-18 05:00:00.000000000
|
||
|
|
+++ convert_hf_to_gguf.py.patched 2025-10-18 06:00:00.000000000
|
||
|
|
@@ -897,6 +897,10 @@
|
||
|
|
if chkhsh == "53e325976a6e142379c19b09afcae354f2f496f147afa8f9e189a33fe4e3024e":
|
||
|
|
# ref: https://huggingface.co/ibm-granite/granite-docling-258M
|
||
|
|
res = "granite-docling"
|
||
|
|
+ if chkhsh == "bf66900d65fe80247e435184a4ac839c5c332657cf567e64b8ede5fbd63f5fd9":
|
||
|
|
+ # Patch for Comma v0.1 tokenizer (Llama 3 compatible)
|
||
|
|
+ # ref: https://huggingface.co/common-pile/comma-v0.1-2t
|
||
|
|
+ res = "llama-bpe"
|
||
|
|
|
||
|
|
if res is None:
|
||
|
|
logger.warning("\n")
|
||
|
|
@@ -911,7 +915,7 @@
|
||
|
|
logger.warning(f"** chkhsh: {chkhsh}")
|
||
|
|
logger.warning("**************************************************************************************")
|
||
|
|
logger.warning("\n")
|
||
|
|
-
|
||
|
|
- raise NotImplementedError("BPE pre-tokenizer was not recognized - update get_vocab_base_pre()")
|
||
|
|
+
|
||
|
|
+ raise NotImplementedError("BPE pre-tokenizer was not recognized - update get_vocab_base_pre()")
|
||
|
|
|
||
|
|
logger.debug(f"tokenizer.ggml.pre: {repr(res)}")
|