commit 9d08fc494b4286f26ec4fb33a33299102b263e7c Author: ModelHub XC Date: Thu Aug 13 03:15:16 2026 +0800 初始化项目，由ModelHub XC社区提供模型 Model: mradermacher/granite-3.1-8b-base-i1-GGUF Source: Original Platform diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..db19ed8 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,58 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +imatrix.dat filter=lfs diff=lfs merge=lfs -text +granite-3.1-8b-base.i1-IQ1_M.gguf filter=lfs diff=lfs merge=lfs -text +granite-3.1-8b-base.i1-IQ1_S.gguf filter=lfs diff=lfs merge=lfs -text +granite-3.1-8b-base.i1-IQ2_M.gguf filter=lfs diff=lfs merge=lfs -text +granite-3.1-8b-base.i1-IQ2_S.gguf filter=lfs diff=lfs merge=lfs -text +granite-3.1-8b-base.i1-IQ2_XS.gguf filter=lfs diff=lfs merge=lfs -text +granite-3.1-8b-base.i1-IQ2_XXS.gguf filter=lfs diff=lfs merge=lfs -text +granite-3.1-8b-base.i1-IQ3_M.gguf filter=lfs diff=lfs merge=lfs -text +granite-3.1-8b-base.i1-IQ3_S.gguf filter=lfs diff=lfs merge=lfs -text +granite-3.1-8b-base.i1-IQ3_XS.gguf filter=lfs diff=lfs merge=lfs -text +granite-3.1-8b-base.i1-IQ3_XXS.gguf filter=lfs diff=lfs merge=lfs -text +granite-3.1-8b-base.i1-IQ4_XS.gguf filter=lfs diff=lfs merge=lfs -text +granite-3.1-8b-base.i1-Q2_K.gguf filter=lfs diff=lfs merge=lfs -text +granite-3.1-8b-base.i1-Q2_K_S.gguf filter=lfs diff=lfs merge=lfs -text +granite-3.1-8b-base.i1-Q3_K_L.gguf filter=lfs diff=lfs merge=lfs -text +granite-3.1-8b-base.i1-Q3_K_M.gguf filter=lfs diff=lfs merge=lfs -text +granite-3.1-8b-base.i1-Q3_K_S.gguf filter=lfs diff=lfs merge=lfs -text +granite-3.1-8b-base.i1-Q4_0.gguf filter=lfs diff=lfs merge=lfs -text +granite-3.1-8b-base.i1-Q4_K_M.gguf filter=lfs diff=lfs merge=lfs -text +granite-3.1-8b-base.i1-Q4_K_S.gguf filter=lfs diff=lfs merge=lfs -text +granite-3.1-8b-base.i1-Q5_K_M.gguf filter=lfs diff=lfs merge=lfs -text +granite-3.1-8b-base.i1-Q5_K_S.gguf filter=lfs diff=lfs merge=lfs -text +granite-3.1-8b-base.i1-Q6_K.gguf filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..07d0173 --- /dev/null +++ b/README.md @@ -0,0 +1,77 @@ +--- +base_model: ibm-granite/granite-3.1-8b-base +language: +- en +library_name: transformers +license: apache-2.0 +quantized_by: mradermacher +tags: +- language +- granite-3.1 +--- +## About + + + + + + +weighted/imatrix quants of https://huggingface.co/ibm-granite/granite-3.1-8b-base + + +static quants are available at https://huggingface.co/mradermacher/granite-3.1-8b-base-GGUF +## Usage + +If you are unsure how to use GGUF files, refer to one of [TheBloke's +READMEs](https://huggingface.co/TheBloke/KafkaLM-70B-German-V0.1-GGUF) for +more details, including on how to concatenate multi-part files. + +## Provided Quants + +(sorted by size, not necessarily quality. IQ-quants are often preferable over similar sized non-IQ quants) + +| Link | Type | Size/GB | Notes | +|:-----|:-----|--------:|:------| +| [GGUF](https://huggingface.co/mradermacher/granite-3.1-8b-base-i1-GGUF/resolve/main/granite-3.1-8b-base.i1-IQ1_S.gguf) | i1-IQ1_S | 1.9 | for the desperate | +| [GGUF](https://huggingface.co/mradermacher/granite-3.1-8b-base-i1-GGUF/resolve/main/granite-3.1-8b-base.i1-IQ1_M.gguf) | i1-IQ1_M | 2.1 | mostly desperate | +| [GGUF](https://huggingface.co/mradermacher/granite-3.1-8b-base-i1-GGUF/resolve/main/granite-3.1-8b-base.i1-IQ2_XXS.gguf) | i1-IQ2_XXS | 2.4 | | +| [GGUF](https://huggingface.co/mradermacher/granite-3.1-8b-base-i1-GGUF/resolve/main/granite-3.1-8b-base.i1-IQ2_XS.gguf) | i1-IQ2_XS | 2.6 | | +| [GGUF](https://huggingface.co/mradermacher/granite-3.1-8b-base-i1-GGUF/resolve/main/granite-3.1-8b-base.i1-IQ2_S.gguf) | i1-IQ2_S | 2.7 | | +| [GGUF](https://huggingface.co/mradermacher/granite-3.1-8b-base-i1-GGUF/resolve/main/granite-3.1-8b-base.i1-IQ2_M.gguf) | i1-IQ2_M | 2.9 | | +| [GGUF](https://huggingface.co/mradermacher/granite-3.1-8b-base-i1-GGUF/resolve/main/granite-3.1-8b-base.i1-Q2_K_S.gguf) | i1-Q2_K_S | 3.0 | very low quality | +| [GGUF](https://huggingface.co/mradermacher/granite-3.1-8b-base-i1-GGUF/resolve/main/granite-3.1-8b-base.i1-Q2_K.gguf) | i1-Q2_K | 3.2 | IQ3_XXS probably better | +| [GGUF](https://huggingface.co/mradermacher/granite-3.1-8b-base-i1-GGUF/resolve/main/granite-3.1-8b-base.i1-IQ3_XXS.gguf) | i1-IQ3_XXS | 3.3 | lower quality | +| [GGUF](https://huggingface.co/mradermacher/granite-3.1-8b-base-i1-GGUF/resolve/main/granite-3.1-8b-base.i1-IQ3_XS.gguf) | i1-IQ3_XS | 3.5 | | +| [GGUF](https://huggingface.co/mradermacher/granite-3.1-8b-base-i1-GGUF/resolve/main/granite-3.1-8b-base.i1-Q3_K_S.gguf) | i1-Q3_K_S | 3.7 | IQ3_XS probably better | +| [GGUF](https://huggingface.co/mradermacher/granite-3.1-8b-base-i1-GGUF/resolve/main/granite-3.1-8b-base.i1-IQ3_S.gguf) | i1-IQ3_S | 3.7 | beats Q3_K* | +| [GGUF](https://huggingface.co/mradermacher/granite-3.1-8b-base-i1-GGUF/resolve/main/granite-3.1-8b-base.i1-IQ3_M.gguf) | i1-IQ3_M | 3.8 | | +| [GGUF](https://huggingface.co/mradermacher/granite-3.1-8b-base-i1-GGUF/resolve/main/granite-3.1-8b-base.i1-Q3_K_M.gguf) | i1-Q3_K_M | 4.1 | IQ3_S probably better | +| [GGUF](https://huggingface.co/mradermacher/granite-3.1-8b-base-i1-GGUF/resolve/main/granite-3.1-8b-base.i1-Q3_K_L.gguf) | i1-Q3_K_L | 4.4 | IQ3_M probably better | +| [GGUF](https://huggingface.co/mradermacher/granite-3.1-8b-base-i1-GGUF/resolve/main/granite-3.1-8b-base.i1-IQ4_XS.gguf) | i1-IQ4_XS | 4.5 | | +| [GGUF](https://huggingface.co/mradermacher/granite-3.1-8b-base-i1-GGUF/resolve/main/granite-3.1-8b-base.i1-Q4_0.gguf) | i1-Q4_0 | 4.8 | fast, low quality | +| [GGUF](https://huggingface.co/mradermacher/granite-3.1-8b-base-i1-GGUF/resolve/main/granite-3.1-8b-base.i1-Q4_K_S.gguf) | i1-Q4_K_S | 4.8 | optimal size/speed/quality | +| [GGUF](https://huggingface.co/mradermacher/granite-3.1-8b-base-i1-GGUF/resolve/main/granite-3.1-8b-base.i1-Q4_K_M.gguf) | i1-Q4_K_M | 5.0 | fast, recommended | +| [GGUF](https://huggingface.co/mradermacher/granite-3.1-8b-base-i1-GGUF/resolve/main/granite-3.1-8b-base.i1-Q5_K_S.gguf) | i1-Q5_K_S | 5.7 | | +| [GGUF](https://huggingface.co/mradermacher/granite-3.1-8b-base-i1-GGUF/resolve/main/granite-3.1-8b-base.i1-Q5_K_M.gguf) | i1-Q5_K_M | 5.9 | | +| [GGUF](https://huggingface.co/mradermacher/granite-3.1-8b-base-i1-GGUF/resolve/main/granite-3.1-8b-base.i1-Q6_K.gguf) | i1-Q6_K | 6.8 | practically like static Q6_K | + +Here is a handy graph by ikawrakow comparing some lower-quality quant +types (lower is better): + +![image.png](https://www.nethype.de/huggingface_embed/quantpplgraph.png) + +And here are Artefact2's thoughts on the matter: +https://gist.github.com/Artefact2/b5f810600771265fc1e39442288e8ec9 + +## FAQ / Model Request + +See https://huggingface.co/mradermacher/model_requests for some answers to +questions you might have and/or if you want some other model quantized. + +## Thanks + +I thank my company, [nethype GmbH](https://www.nethype.de/), for letting +me use its servers and providing upgrades to my workstation to enable +this work in my free time. Additional thanks to [@nicoboss](https://huggingface.co/nicoboss) for giving me access to his private supercomputer, enabling me to provide many more imatrix quants, at much higher quality, than I would otherwise be able to. + + diff --git a/granite-3.1-8b-base.i1-IQ1_M.gguf b/granite-3.1-8b-base.i1-IQ1_M.gguf new file mode 100644 index 0000000..3d97e80 --- /dev/null +++ b/granite-3.1-8b-base.i1-IQ1_M.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4fb1f80af003af2ca862acb98e327e340208fa154a6eb3cbc951e79abfbe8c43 +size 1997297120 diff --git a/granite-3.1-8b-base.i1-IQ1_S.gguf b/granite-3.1-8b-base.i1-IQ1_S.gguf new file mode 100644 index 0000000..b81e526 --- /dev/null +++ b/granite-3.1-8b-base.i1-IQ1_S.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:63391742c58259431e9fc500ae426ac04c57c629be7eccb56c848010e85a6bb3 +size 1836324320 diff --git a/granite-3.1-8b-base.i1-IQ2_M.gguf b/granite-3.1-8b-base.i1-IQ2_M.gguf new file mode 100644 index 0000000..30e4003 --- /dev/null +++ b/granite-3.1-8b-base.i1-IQ2_M.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0135a225d671567cece3f242257fe243d854e37a4096a2fbf0807703acc73c84 +size 2836813280 diff --git a/granite-3.1-8b-base.i1-IQ2_S.gguf b/granite-3.1-8b-base.i1-IQ2_S.gguf new file mode 100644 index 0000000..b547f3f --- /dev/null +++ b/granite-3.1-8b-base.i1-IQ2_S.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b2b2197cc96592dd19e34cfe476705f53dad1a5720e7c9049788320debb92e4e +size 2622182880 diff --git a/granite-3.1-8b-base.i1-IQ2_XS.gguf b/granite-3.1-8b-base.i1-IQ2_XS.gguf new file mode 100644 index 0000000..4097f29 --- /dev/null +++ b/granite-3.1-8b-base.i1-IQ2_XS.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6a44eb483d176369199e43f7c6939303f881062bdacd921e17d4e1ef56eb90be +size 2501187040 diff --git a/granite-3.1-8b-base.i1-IQ2_XXS.gguf b/granite-3.1-8b-base.i1-IQ2_XXS.gguf new file mode 100644 index 0000000..1f633cc --- /dev/null +++ b/granite-3.1-8b-base.i1-IQ2_XXS.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1fae4f77f4e5e1fdab3b1c5cf4bab8656bd89e912f83c0de69b83fe018d97651 +size 2265585120 diff --git a/granite-3.1-8b-base.i1-IQ3_M.gguf b/granite-3.1-8b-base.i1-IQ3_M.gguf new file mode 100644 index 0000000..7422200 --- /dev/null +++ b/granite-3.1-8b-base.i1-IQ3_M.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fa09f2e3db9d665f5bf0acba37a142f884847215d419ef2c5d9765fe4abe3732 +size 3738703328 diff --git a/granite-3.1-8b-base.i1-IQ3_S.gguf b/granite-3.1-8b-base.i1-IQ3_S.gguf new file mode 100644 index 0000000..f4cee34 --- /dev/null +++ b/granite-3.1-8b-base.i1-IQ3_S.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1d8fdf7c146680ff5ca83f23f0bed00005ef177187d8210be0e52181dbea8fa0 +size 3614758368 diff --git a/granite-3.1-8b-base.i1-IQ3_XS.gguf b/granite-3.1-8b-base.i1-IQ3_XS.gguf new file mode 100644 index 0000000..161b205 --- /dev/null +++ b/granite-3.1-8b-base.i1-IQ3_XS.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2f58c9502565d9f6335bc19324e7a4bb682fbe79c732832eb5173089a70940f7 +size 3427980768 diff --git a/granite-3.1-8b-base.i1-IQ3_XXS.gguf b/granite-3.1-8b-base.i1-IQ3_XXS.gguf new file mode 100644 index 0000000..3dc1e51 --- /dev/null +++ b/granite-3.1-8b-base.i1-IQ3_XXS.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eb046e900d9e2f7a1f3618e618409b8990c110c03f9a2641757215e741281c7b +size 3201357280 diff --git a/granite-3.1-8b-base.i1-IQ4_XS.gguf b/granite-3.1-8b-base.i1-IQ4_XS.gguf new file mode 100644 index 0000000..4122bfb --- /dev/null +++ b/granite-3.1-8b-base.i1-IQ4_XS.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b899b7a3fea9d00135ce5ac4c7f3b56aa645b320b72c13c6c51b5f725f684927 +size 4428060128 diff --git a/granite-3.1-8b-base.i1-Q2_K.gguf b/granite-3.1-8b-base.i1-Q2_K.gguf new file mode 100644 index 0000000..6d7b561 --- /dev/null +++ b/granite-3.1-8b-base.i1-Q2_K.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:48c5f993a9d65e8b15e82029d86833ca29ec552dd1e4f12b35c735b0e27c783c +size 3103577568 diff --git a/granite-3.1-8b-base.i1-Q2_K_S.gguf b/granite-3.1-8b-base.i1-Q2_K_S.gguf new file mode 100644 index 0000000..e614ccc --- /dev/null +++ b/granite-3.1-8b-base.i1-Q2_K_S.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6b24a088a9219d964e288e2058f35052e729001244bfc51c67a792dc928c877d +size 2883868128 diff --git a/granite-3.1-8b-base.i1-Q3_K_L.gguf b/granite-3.1-8b-base.i1-Q3_K_L.gguf new file mode 100644 index 0000000..6bbb269 --- /dev/null +++ b/granite-3.1-8b-base.i1-Q3_K_L.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2bca9b17001c2ff0207a8342d8fa9620c887e6fc746d39923bb878c01bce0a3b +size 4349416928 diff --git a/granite-3.1-8b-base.i1-Q3_K_M.gguf b/granite-3.1-8b-base.i1-Q3_K_M.gguf new file mode 100644 index 0000000..1176dcd --- /dev/null +++ b/granite-3.1-8b-base.i1-Q3_K_M.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e60c8dc7e2aeab471117004f2095ac56b97e6225cafd8d58b5881bb59511b209 +size 3996571104 diff --git a/granite-3.1-8b-base.i1-Q3_K_S.gguf b/granite-3.1-8b-base.i1-Q3_K_S.gguf new file mode 100644 index 0000000..ea8467f --- /dev/null +++ b/granite-3.1-8b-base.i1-Q3_K_S.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:89d74688295a8bc90d182e1e08055e7f18db74d18e96a5b509dabe70b1fe1d03 +size 3592476128 diff --git a/granite-3.1-8b-base.i1-Q4_0.gguf b/granite-3.1-8b-base.i1-Q4_0.gguf new file mode 100644 index 0000000..99a00b3 --- /dev/null +++ b/granite-3.1-8b-base.i1-Q4_0.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:12c662f56d01f6d318745b1c4a426fd5126cad4fbb4df00aaafa9b049242bde4 +size 4667266528 diff --git a/granite-3.1-8b-base.i1-Q4_K_M.gguf b/granite-3.1-8b-base.i1-Q4_K_M.gguf new file mode 100644 index 0000000..5ed17fc --- /dev/null +++ b/granite-3.1-8b-base.i1-Q4_K_M.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:37265168724838d229c2b79410136b12a39589f2075e1e776bd699cb0ce8f51b +size 4942845408 diff --git a/granite-3.1-8b-base.i1-Q4_K_S.gguf b/granite-3.1-8b-base.i1-Q4_K_S.gguf new file mode 100644 index 0000000..9ca9028 --- /dev/null +++ b/granite-3.1-8b-base.i1-Q4_K_S.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6940d8de3ef0da6ffb11b5f66f1f1dc8c6ed3deaf06376260f41d7357c599bcf +size 4685747680 diff --git a/granite-3.1-8b-base.i1-Q5_K_M.gguf b/granite-3.1-8b-base.i1-Q5_K_M.gguf new file mode 100644 index 0000000..c5af805 --- /dev/null +++ b/granite-3.1-8b-base.i1-Q5_K_M.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:67b02ed256f0acbc10b958a8543c348a8e01744c8e23c97c510bab68c105fac9 +size 5797434848 diff --git a/granite-3.1-8b-base.i1-Q5_K_S.gguf b/granite-3.1-8b-base.i1-Q5_K_S.gguf new file mode 100644 index 0000000..e7949ff --- /dev/null +++ b/granite-3.1-8b-base.i1-Q5_K_S.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bd1e928bfd39df24df1ae4485fece66d0da7bcaee02f365cde9a4a14d67e8b0c +size 5647029728 diff --git a/granite-3.1-8b-base.i1-Q6_K.gguf b/granite-3.1-8b-base.i1-Q6_K.gguf new file mode 100644 index 0000000..fbd570d --- /dev/null +++ b/granite-3.1-8b-base.i1-Q6_K.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c79ad43f197066fa40b11c924cd89641a3cd5882546adc85ad86dc873882a779 +size 6705436128 diff --git a/imatrix.dat b/imatrix.dat new file mode 100644 index 0000000..d47a9df --- /dev/null +++ b/imatrix.dat @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:88414b0587c6b77b8826eebca69638ea6fc4e41547b72bf583f6cd36c2ec51d7 +size 5989445