commit 9e2923a59dad028d102931de90da1fe6ad4f6472 Author: ModelHub XC Date: Sat Sep 19 23:58:31 2026 +0800 初始化项目，由ModelHub XC社区提供模型 Model: Melvin56/Qwen3-4B-ik_GGUF Source: Original Platform diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..e9f3f4e --- /dev/null +++ b/.gitattributes @@ -0,0 +1,49 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +qwen3-4b-IQ6_K.gguf filter=lfs diff=lfs merge=lfs -text +qwen3-4b-IQ5_K.gguf filter=lfs diff=lfs merge=lfs -text +qwen3-4b-IQ4_K.gguf filter=lfs diff=lfs merge=lfs -text +qwen3-4b-IQ4_KS.gguf filter=lfs diff=lfs merge=lfs -text +imatrix.dat filter=lfs diff=lfs merge=lfs -text +qwen3-4b-BF16.gguf filter=lfs diff=lfs merge=lfs -text +iimatrix.dat filter=lfs diff=lfs merge=lfs -text +qwen3-4b-IQ6_K-unsloth.gguf filter=lfs diff=lfs merge=lfs -text +1imatrix.dat filter=lfs diff=lfs merge=lfs -text +qwen3-4b-Q4_0_R8.gguf filter=lfs diff=lfs merge=lfs -text +qwen3-4b-0IQ4_K.gguf filter=lfs diff=lfs merge=lfs -text +qwen3-4b-0IQ4_KS.gguf filter=lfs diff=lfs merge=lfs -text +qwen3-4b-IQ3_K.gguf filter=lfs diff=lfs merge=lfs -text +qwen3-4b-IQ4_K01.gguf filter=lfs diff=lfs merge=lfs -text diff --git a/Qwen3-4B.png b/Qwen3-4B.png new file mode 100644 index 0000000..c1bc8d1 Binary files /dev/null and b/Qwen3-4B.png differ diff --git a/README.md b/README.md new file mode 100644 index 0000000..329fddf --- /dev/null +++ b/README.md @@ -0,0 +1,69 @@ +--- +license: apache-2.0 +license_link: https://huggingface.co/Qwen/Qwen3-4B/blob/main/LICENSE +pipeline_tag: text-generation +base_model: +- Qwen/Qwen3-4B +language: +- en +tags: +- imatrix +- qwen3 +- conversational +--- +# Melvin56/Qwen3-4B-ik_GGUF + +# Quant for [ik_llama.cpp](https://github.com/ikawrakow/ik_llama.cpp) + +Build: 3680 (a2d24c97) + +Original Model : [Qwen/Qwen3-4B](https://huggingface.co/Qwen/Qwen3-4B) + +I used imatrix to create all these quants using this [Dataset](https://gist.github.com/tristandruyen/9e207a95c7d75ddf37525d353e00659c/#file-calibration_data_v5_rc-txt). + +Perplexity + +
+Perplexity measurement methodology. + +I tested all quants using ik_llama.cpp build 3680 (a2d24c97) +``` +ik_llama.cpp/build/bin/llama-perplexity \ +-m .gguf \ +--ctx-size 512 \ +--ubatch-size 512 \ +-f wikitext-2-raw/wiki.test.raw \ +-fa \ +-ngl 999 +``` + +# Raw data + +| Quant | Size (GB) | PPL | +|---------|-----------|---------------------| +| BF16 | 8.05 | 14.3308 +/- 0.13259 | +| IQ6_K | 3.34 | 14.2810 +/- 0.13159 | +| IQ5_K | 2.82 | 14.5004 +/- 0.13465 | +| IQ4_K | 2.38 | 14.5280 +/- 0.13414 | +| IQ4_KS | 2.22 | 15.2121 +/- 0.14294 | + +
+ +--- + +| | CPU (AVX2) | CPU (ARM NEON) | Metal | cuBLAS | rocBLAS | SYCL | CLBlast | Vulkan | Kompute | +| :------------ | :---------: | :------------: | :---: | :----: | :-----: | :---: | :------: | :----: | :------: | +| K-quants | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ 🐢5 | ✅ 🐢5 | ❌ | +| I-quants | ✅ 🐢4 | ✅ 🐢4 | ✅ 🐢4 | ✅ | ✅ | Partial¹ | ❌ | ❌ | ❌ | +``` +✅: feature works +🚫: feature does not work +❓: unknown, please contribute if you can test it youself +🐢: feature is slow +¹: IQ3_S and IQ1_S, see #5886 +²: Only with -ngl 0 +³: Inference is 50% slower +⁴: Slower than K-quants of comparable size +⁵: Slower than cuBLAS/rocBLAS on similar cards +⁶: Only q8_0 and iq4_nl +``` \ No newline at end of file diff --git a/qwen3-4b-0IQ4_K.gguf b/qwen3-4b-0IQ4_K.gguf new file mode 100644 index 0000000..77c01e0 --- /dev/null +++ b/qwen3-4b-0IQ4_K.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d0480fd63f260ef72520b11b13356d1ab6fdead7c62ab2afda8101098a69a93d +size 2381343648 diff --git a/qwen3-4b-0IQ4_KS.gguf b/qwen3-4b-0IQ4_KS.gguf new file mode 100644 index 0000000..fc5f498 --- /dev/null +++ b/qwen3-4b-0IQ4_KS.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2f899f3ea3edf390a1124d53931ba882d57ace12314e1686d68305007ceb92eb +size 2223369632 diff --git a/qwen3-4b-BF16.gguf b/qwen3-4b-BF16.gguf new file mode 100644 index 0000000..c3f315d --- /dev/null +++ b/qwen3-4b-BF16.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e9698fb4734ae2b921676793ac1e05dad88fde774ed1c093d12204c76e164aeb +size 8051285120 diff --git a/qwen3-4b-IQ4_K.gguf b/qwen3-4b-IQ4_K.gguf new file mode 100644 index 0000000..10650f3 --- /dev/null +++ b/qwen3-4b-IQ4_K.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:341ad59118439139f5a9286c0a4eeca9285b006b96135fa77d09dfe543a50cda +size 2478664608 diff --git a/qwen3-4b-IQ4_K01.gguf b/qwen3-4b-IQ4_K01.gguf new file mode 100644 index 0000000..4d15b5a --- /dev/null +++ b/qwen3-4b-IQ4_K01.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cf34e0f430f29594508893786f5d5492e7d820e4dbd844710af2ddd36021d594 +size 2620222368 diff --git a/qwen3-4b-IQ5_K.gguf b/qwen3-4b-IQ5_K.gguf new file mode 100644 index 0000000..4478c28 --- /dev/null +++ b/qwen3-4b-IQ5_K.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b3e88a9916a75a5bc12a7e85af80de8982416c9e1b47b1b322f706f6dc6bd88c +size 2823711648 diff --git a/qwen3-4b-IQ6_K-unsloth.gguf b/qwen3-4b-IQ6_K-unsloth.gguf new file mode 100644 index 0000000..4f02bf2 --- /dev/null +++ b/qwen3-4b-IQ6_K-unsloth.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dc49463e20b0ffb2140f41e26ddaf1084ab97ac255b68b0e5e6e38b01b0c765f +size 3337685376 diff --git a/qwen3-4b-IQ6_K.gguf b/qwen3-4b-IQ6_K.gguf new file mode 100644 index 0000000..4c23cf8 --- /dev/null +++ b/qwen3-4b-IQ6_K.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4b7d9e32f16ef1913fcf6361445305cfbaf01a5ebb20a7f15e9fcf0882dedb61 +size 3337685408 diff --git a/qwen3-4b-Q4_0_R8.gguf b/qwen3-4b-Q4_0_R8.gguf new file mode 100644 index 0000000..2f31cb0 --- /dev/null +++ b/qwen3-4b-Q4_0_R8.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15710044003d8e6e87ae5543139574132b3644f448ba00a32fd0bf0369e9dd4e +size 2369547168