From 924e60a42638a486a4e224dd3e233b44bb5cecdd Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Mon, 14 Sep 2026 04:52:16 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: ymcki/gemma-2-9b-it-GGUF Source: Original Platform --- .gitattributes | 46 ++++++++ README.md | 159 ++++++++++++++++++++++++++++ gemma-2-9b-it-imatrix.Q4_0.gguf | 3 + gemma-2-9b-it-imatrix.Q4_0_4_4.gguf | 3 + gemma-2-9b-it-imatrix.Q4_0_4_8.gguf | 3 + gemma-2-9b-it-imatrix.Q4_0_8_8.gguf | 3 + gemma-2-9b-it.Q4_0.gguf | 3 + gemma-2-9b-it.Q4_0_4_4.gguf | 3 + gemma-2-9b-it.Q4_0_4_8.gguf | 3 + gemma-2-9b-it.Q4_0_8_8.gguf | 3 + gemma-2-9b-it.Q8_0.gguf | 3 + gemma-2-9b-it.f16.gguf | 3 + gemma-2-9b-it.imatrix | 3 + 13 files changed, 238 insertions(+) create mode 100644 .gitattributes create mode 100644 README.md create mode 100644 gemma-2-9b-it-imatrix.Q4_0.gguf create mode 100644 gemma-2-9b-it-imatrix.Q4_0_4_4.gguf create mode 100644 gemma-2-9b-it-imatrix.Q4_0_4_8.gguf create mode 100644 gemma-2-9b-it-imatrix.Q4_0_8_8.gguf create mode 100644 gemma-2-9b-it.Q4_0.gguf create mode 100644 gemma-2-9b-it.Q4_0_4_4.gguf create mode 100644 gemma-2-9b-it.Q4_0_4_8.gguf create mode 100644 gemma-2-9b-it.Q4_0_8_8.gguf create mode 100644 gemma-2-9b-it.Q8_0.gguf create mode 100644 gemma-2-9b-it.f16.gguf create mode 100644 gemma-2-9b-it.imatrix diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..b02b3e4 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,46 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +gemma-2-9b-it.f16.gguf filter=lfs diff=lfs merge=lfs -text +gemma-2-9b-it.Q8_0.gguf filter=lfs diff=lfs merge=lfs -text +gemma-2-9b-it.imatrix filter=lfs diff=lfs merge=lfs -text +gemma-2-9b-it-imatrix.Q4_0.gguf filter=lfs diff=lfs merge=lfs -text +gemma-2-9b-it.Q4_0.gguf filter=lfs diff=lfs merge=lfs -text +gemma-2-9b-it-imatrix.Q4_0_4_4.gguf filter=lfs diff=lfs merge=lfs -text +gemma-2-9b-it.Q4_0_4_4.gguf filter=lfs diff=lfs merge=lfs -text +gemma-2-9b-it-imatrix.Q4_0_4_8.gguf filter=lfs diff=lfs merge=lfs -text +gemma-2-9b-it.Q4_0_4_8.gguf filter=lfs diff=lfs merge=lfs -text +gemma-2-9b-it-imatrix.Q4_0_8_8.gguf filter=lfs diff=lfs merge=lfs -text +gemma-2-9b-it.Q4_0_8_8.gguf filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..fee95fd --- /dev/null +++ b/README.md @@ -0,0 +1,159 @@ +--- +base_model: google/gemma-2-9b-it +language: +- multilingual +datasets: + - TFMC/imatrix-dataset-for-japanese-llm +library_name: transformers +license: gemma +license_link: https://ai.google.dev/gemma/terms +pipeline_tag: text-generation +tags: +- nlp +- code +quantized_by: ymcki +widget: +- messages: + - role: user + content: Can you provide ways to eat combinations of bananas and dragonfruits? +--- + +Original model: https://huggingface.co/google/gemma-2-9b-it + +## Description + +The purpose of this repository is to see whether Japanese specific +imatrix can improve the performance of a non Japanese optimized model. + +It also provides the Q4_0_8_8, Q4_0_4_8 and Q4_0_4_4 ggufs for edge +devices that were otherwise not made by bartowski. These models should +also be good for edge devices with 16GB RAM. + +## Prompt format + +``` +user +{prompt} +model + +model + +``` + +Note that this model does not support a System prompt. + +## Download a file (not the whole branch) from below: + +ELIZA-Tasks-100 is pretty standard benchmark for Japanese LLMs. +The perfect score is 5.00. As a reference, bartowski's gemma-2-27b-it.Q6_K.gguf scores 4.04. + +| Filename | Quant type | File Size | ELIZA-Tasks-100 | Nvidia 3090 | Description | +| -------- | ---------- | --------- | --------------- | ----------- | ----------- | +| [gemma-2-9b-it.f16.gguf](https://huggingface.co/ymcki/gemma-2-9b-it-GGUF/blob/main/gemma-2-9b-it.f16.gguf) | f16 | 18.5GB | 3.75 | 31.9t/s | Full F16 weights. | +| [gemma-2-9b-it.Q8_0.gguf](https://huggingface.co/ymcki/gemma-2-9b-it-GGUF/blob/main/gemma-2-9b-it.Q8_0.gguf) | Q8_0 | 9.83GB | 3.66 | 56.1t/s | Extremely high quality, *recommended for edge devices with 16GB RAM*. | +| [gemma-2-9b-it-imatrix.Q4_0.gguf](https://huggingface.co/ymcki/gemma-2-9b-it-GGUF/blob/main/gemma-2-9b-it-imatrix.Q4_0.gguf) | Q4_0 | 5.44GB | 3.76 | 80.6t/s | Good quality, *recommended for edge devices wth 8GB RAM*. | +| [gemma-2-9b-it-imatrix.Q4_0_8_8.gguf](https://huggingface.co/ymcki/gemma-2-9b-it-GGUF/blob/main/gemma-2-9b-it-imatrix.Q4_0_8_8.gguf) | Q4_0_8_8 | 5.44GB | 3.74 | 0.7t/s | Good quality, *recommended for edge devices with 8GB RAM*. | +| [gemma-2-9b-it-imatrix.Q4_0_4_8.gguf](https://huggingface.co/ymcki/gemma-2-9b-it-GGUF/blob/main/gemma-2-9b-it-imatrix.Q4_0_4_8.gguf) | Q4_0_4_8 | 5.44GB | 3.64 | 0.7t/s | Good quality, *recommended for edge devices with 8GB RAM*. | +| [gemma-2-9b-it-imatrix.Q4_0_4_4.gguf](https://huggingface.co/ymcki/gemma-2-9b-it-GGUF/blob/main/gemma-2-9b-it-imatrix.Q4_0_4_4.gguf) | Q4_0_4_4 | 5.44GB | 3.72 | 0.72t/s | Good quality, *recommended for edge devices with 8GB RAM*. | +| [gemma-2-9b-it.Q4_0.gguf](https://huggingface.co/ymcki/gemma-2-9b-it-GGUF/blob/main/gemma-2-9b-it.Q4_0.gguf) | Q4_0 | 5.44GB | 3.64 | 65.1t/s | Good quality, *recommended for edge device with 8GB RAM* | +| [gemma-2-9b-it.Q4_0_8_8.gguf](https://huggingface.co/ymcki/gemma-2-9b-it-GGUF/blob/main/gemma-2-9b-it.Q4_0_8_8.gguf) | Q4_0_8_8 | 5.44GB | 3.64 | 0.57t/s | Good quality but imatrix version seems better. | +| [gemma-2-9b-it.Q4_0_4_8.gguf](https://huggingface.co/ymcki/gemma-2-9b-it-GGUF/blob/main/gemma-2-9b-it.Q4_0_4_8.gguf) | Q4_0_4_8 | 5.44GB | 3.68 | 0.61t/s | Good quality but imatrix version seems better. | +| [gemma-2-9b-it.Q4_0_4_4.gguf](https://huggingface.co/ymcki/gemma-2-9b-it-GGUF/blob/main/gemma-2-9b-it.Q4_0_4_4.gguf) | Q4_0_4_4 | 5.44GB | 3.63 | 0.76t/s | Good quality but imatrix version seems better. | + +## How to check i8mm and sve support for ARM devices + +ARM i8mm support is necessary to take advantage of Q4_0_4_8 gguf. All ARM architecture >= ARMv8.6-A supports i8mm. + +ARM sve support is necessary to take advantage of Q4_0_8_8 gguf. sve is an optional feature that starts from ARMv8.2-A but majority of ARM chips doesn't implement it. + +For ARM devices without both, it is recommended to use Q4_0_4_4. However, in reality, Q4_0 can perform better for some phones, so you better try both and see which one is better. + +With these support, the inference speed should be faster in the order of Q4_0_8_8 > Q4_0_4_8 > Q4_0_4_4 > Q4_0 without much effect on the quality of response. + +This is a [list](https://gpages.juszkiewicz.com.pl/arm-socs-table/arm-socs.html) of ARM CPUs that support different ARM instructions. Another [list](https://raw.githubusercontent.com/ThomasKaiser/sbc-bench/refs/heads/master/sbc-bench.sh). Apparently, they only cover limited number of ARM CPUs. It is better you check for i8mm and sve support by yourself. + +For Apple devices, + +``` +sysctl hw +``` + +For other ARM devices (ie most Android devices), +``` +cat /proc/cpuinfo +``` + +There are also android apps that can display /proc/cpuinfo. + +I was told that for Intel/AMD CPU inference, support for AVX2/AVX512 can also improve the performance of Q4_0_8_8. + +On the other hand, Nvidia 3090 inference speed is significantly faster for Q4_0 than the other ggufs. That means for GPU inference, you better off using Q4_0. + +## Which Q4_0 model to use for ARM devices +| Brand | Series | Model | i8mm | sve | Quant Type | +| ----- | ------ | ----- | ---- | --- | -----------| +| Apple | A | A4 to A14 | No | No | Q4_0_4_4 | +| Apple | A | A15 to A18 | Yes | No | Q4_0_4_8 | +| Apple | M | M1 | No | No | Q4_0_4_4 | +| Apple | M | M2/M3/M4 | Yes | No | Q4_0_4_8 | +| Google | Tensor | G1,G2 | No | No | Q4_0_4_4 | +| Google | Tensor | G3,G4 | Yes | Yes | Q4_0_8_8 | +| Samsung | Exynos | 2200,2400 | Yes | Yes | Q4_0_8_8 | +| Mediatek | Dimensity | 9000,9000+ | Yes | Yes | Q4_0_8_8 | +| Mediatek | Dimensity | 9300 | Yes | No | Q4_0_4_8 | +| Qualcomm | Snapdragon | 7+ Gen 2,8/8+ Gen 1 | Yes | Yes | Q4_0_8_8 | +| Qualcomm | Snapdragon | 8 Gen 2,8 Gen 3,X Elite | Yes | No | Q4_0_4_8 | + +## imatrix quantization + +According to this [blog](https://sc-bakushu.hatenablog.com/entry/2024/04/20/050213), adding imatrix to low bit quant can significantly improve performance. The best dataset for Japanese is [MTFMC/imatrix-dataset-for-japanese-llm](https://huggingface.co/datasets/TFMC/imatrix-dataset-for-japanese-llm). Therefore, I also created the imatrix versions of different Q4_0 quants. + +However, based on my benchmarking results, it seems like imatrix does improve the performance of a non-Japanese optimized model but doesn't do much for a Japanese optimized model like [gemma-2-2b-jpn-it](https://huggingface.co/ymcki/gemma-2-2b-jpn-it-GGUF/). + +## Convert safetensors to f16 gguf + +Make sure you have llama.cpp git cloned: + +``` +python3 convert_hf_to_gguf.py gemma-2-9b-it/ --outfile gemma-2-9b-it.f16.gguf --outtype f16 +``` + +## Convert f16 gguf to Q8_0 gguf without imatrix +Make sure you have llama.cpp compiled: +``` +./llama-quantize gemma-2-9b-it.f16.gguf gemma-2-9b-it.Q8_0.gguf q8_0 +``` + +## Convert f16 gguf to other ggufs with imatrix + +First, prepare imatrix from f16 gguf and c4_en_ja_imatrix.txt + +``` +./llama-imatrix -m gemma-2-9b-it.f16.gguf -f c4_en_ja_imatrix.txt -o gemma-2-9b-it.imatrix --chunks 32 +``` + +Then, convert f16 gguf with imatrix to create imatrix gguf + +``` +./llama-quantize --imatrix gemma-2-9b-it.imatrix gemma-2-9b-it.f16.gguf gemma-2-9b-it-imatrix.Q4_0_8_8.gguf q4_0_8_8 +``` + +## Downloading using huggingface-cli + +First, make sure you have hugginface-cli installed: + +``` +pip install -U "huggingface_hub[cli]" +``` + +Then, you can target the specific file you want: + +``` +huggingface-cli download ymcki/gemma-2-9b-it-GGUF --include "gemma-2-9b-it.Q8_0.gguf" --local-dir ./ +``` + +## Credits + +Thank you bartowski for providing a README.md to get me started. + +Thank you YoutechA320U for the ELYZA-tasks-100 auto evaluation tool. diff --git a/gemma-2-9b-it-imatrix.Q4_0.gguf b/gemma-2-9b-it-imatrix.Q4_0.gguf new file mode 100644 index 0000000..ad71eba --- /dev/null +++ b/gemma-2-9b-it-imatrix.Q4_0.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ea67e48d6fdefb162b625619a151437228c186ffd7495a6677534794f3e90871 +size 5459199008 diff --git a/gemma-2-9b-it-imatrix.Q4_0_4_4.gguf b/gemma-2-9b-it-imatrix.Q4_0_4_4.gguf new file mode 100644 index 0000000..4498dd0 --- /dev/null +++ b/gemma-2-9b-it-imatrix.Q4_0_4_4.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4d51f5c0fb36e780fb1e8b944d85eefc4413dbcc7c0b1293e45395083ac33a4a +size 5443142688 diff --git a/gemma-2-9b-it-imatrix.Q4_0_4_8.gguf b/gemma-2-9b-it-imatrix.Q4_0_4_8.gguf new file mode 100644 index 0000000..2ab1094 --- /dev/null +++ b/gemma-2-9b-it-imatrix.Q4_0_4_8.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9dd025f9a517006eca3d9c948d116338ae60496f29880b03ca66dce24425fbb6 +size 5443142688 diff --git a/gemma-2-9b-it-imatrix.Q4_0_8_8.gguf b/gemma-2-9b-it-imatrix.Q4_0_8_8.gguf new file mode 100644 index 0000000..ff47e46 --- /dev/null +++ b/gemma-2-9b-it-imatrix.Q4_0_8_8.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:32059e28ca182b72ee999eef7f2a4c780be5c1a01a47864928eb1328c8f889ee +size 5443142688 diff --git a/gemma-2-9b-it.Q4_0.gguf b/gemma-2-9b-it.Q4_0.gguf new file mode 100644 index 0000000..05da90e --- /dev/null +++ b/gemma-2-9b-it.Q4_0.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cfc36bb08813f7f508cfd397e44f87a3d59110c47df3e23722434dff84e42077 +size 5443142464 diff --git a/gemma-2-9b-it.Q4_0_4_4.gguf b/gemma-2-9b-it.Q4_0_4_4.gguf new file mode 100644 index 0000000..8b4520d --- /dev/null +++ b/gemma-2-9b-it.Q4_0_4_4.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:48574c0130f1536c61b8b7d8d2511b5faf4eb36bf266b31e4c4248c4b7da04ed +size 5443142464 diff --git a/gemma-2-9b-it.Q4_0_4_8.gguf b/gemma-2-9b-it.Q4_0_4_8.gguf new file mode 100644 index 0000000..ac6bba8 --- /dev/null +++ b/gemma-2-9b-it.Q4_0_4_8.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:80fcbcdd4f94cbcfccb39b3bf0933beaa19544c37aa4901f21abfeeb159ad4e3 +size 5443142464 diff --git a/gemma-2-9b-it.Q4_0_8_8.gguf b/gemma-2-9b-it.Q4_0_8_8.gguf new file mode 100644 index 0000000..0e56cdb --- /dev/null +++ b/gemma-2-9b-it.Q4_0_8_8.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bd3c3fdbcc1048ea1640127b581507df15d30c58affad9d039546c4340f8790e +size 5443142464 diff --git a/gemma-2-9b-it.Q8_0.gguf b/gemma-2-9b-it.Q8_0.gguf new file mode 100644 index 0000000..542b923 --- /dev/null +++ b/gemma-2-9b-it.Q8_0.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3d3913a08ecbf912d8c2b37022cd2917f3deb34c7e3bd7d7340aa7370a31115c +size 9827148608 diff --git a/gemma-2-9b-it.f16.gguf b/gemma-2-9b-it.f16.gguf new file mode 100644 index 0000000..0071914 --- /dev/null +++ b/gemma-2-9b-it.f16.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a8667da7b45bd519e38cedbfb7669cb373d4f9397aa241fecb1580051dad0493 +size 18490680128 diff --git a/gemma-2-9b-it.imatrix b/gemma-2-9b-it.imatrix new file mode 100644 index 0000000..b1302b8 --- /dev/null +++ b/gemma-2-9b-it.imatrix @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:320e9ba88ebbe5ab741a9b7c56c8a41ef2d69de27ab1c7318331dce982c8198d +size 6116887