commit 7b1c841450778c8da2c14a86e75d93f6579f128c Author: ModelHub XC Date: Mon May 4 17:09:11 2026 +0800 初始化项目，由ModelHub XC社区提供模型 Model: mradermacher/FastLlama-3.2-1B-Instruct-i1-GGUF Source: Original Platform diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..2daeeb4 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,61 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +imatrix.dat filter=lfs diff=lfs merge=lfs -text +FastLlama-3.2-1B-Instruct.i1-Q2_K.gguf filter=lfs diff=lfs merge=lfs -text +FastLlama-3.2-1B-Instruct.i1-IQ3_M.gguf filter=lfs diff=lfs merge=lfs -text +FastLlama-3.2-1B-Instruct.i1-Q4_K_S.gguf filter=lfs diff=lfs merge=lfs -text +FastLlama-3.2-1B-Instruct.i1-IQ3_XXS.gguf filter=lfs diff=lfs merge=lfs -text +FastLlama-3.2-1B-Instruct.i1-Q3_K_M.gguf filter=lfs diff=lfs merge=lfs -text +FastLlama-3.2-1B-Instruct.i1-Q4_K_M.gguf filter=lfs diff=lfs merge=lfs -text +FastLlama-3.2-1B-Instruct.i1-IQ2_M.gguf filter=lfs diff=lfs merge=lfs -text +FastLlama-3.2-1B-Instruct.i1-Q6_K.gguf filter=lfs diff=lfs merge=lfs -text +FastLlama-3.2-1B-Instruct.i1-IQ4_XS.gguf filter=lfs diff=lfs merge=lfs -text +FastLlama-3.2-1B-Instruct.i1-Q2_K_S.gguf filter=lfs diff=lfs merge=lfs -text +FastLlama-3.2-1B-Instruct.i1-IQ1_M.gguf filter=lfs diff=lfs merge=lfs -text +FastLlama-3.2-1B-Instruct.i1-Q3_K_S.gguf filter=lfs diff=lfs merge=lfs -text +FastLlama-3.2-1B-Instruct.i1-IQ2_XXS.gguf filter=lfs diff=lfs merge=lfs -text +FastLlama-3.2-1B-Instruct.i1-Q3_K_L.gguf filter=lfs diff=lfs merge=lfs -text +FastLlama-3.2-1B-Instruct.i1-IQ2_XS.gguf filter=lfs diff=lfs merge=lfs -text +FastLlama-3.2-1B-Instruct.i1-Q5_K_S.gguf filter=lfs diff=lfs merge=lfs -text +FastLlama-3.2-1B-Instruct.i1-IQ2_S.gguf filter=lfs diff=lfs merge=lfs -text +FastLlama-3.2-1B-Instruct.i1-Q4_0_4_4.gguf filter=lfs diff=lfs merge=lfs -text +FastLlama-3.2-1B-Instruct.i1-IQ1_S.gguf filter=lfs diff=lfs merge=lfs -text +FastLlama-3.2-1B-Instruct.i1-Q5_K_M.gguf filter=lfs diff=lfs merge=lfs -text +FastLlama-3.2-1B-Instruct.i1-Q4_0.gguf filter=lfs diff=lfs merge=lfs -text +FastLlama-3.2-1B-Instruct.i1-IQ3_XS.gguf filter=lfs diff=lfs merge=lfs -text +FastLlama-3.2-1B-Instruct.i1-Q4_0_4_8.gguf filter=lfs diff=lfs merge=lfs -text +FastLlama-3.2-1B-Instruct.i1-Q4_0_8_8.gguf filter=lfs diff=lfs merge=lfs -text +FastLlama-3.2-1B-Instruct.i1-IQ3_S.gguf filter=lfs diff=lfs merge=lfs -text diff --git a/FastLlama-3.2-1B-Instruct.i1-IQ1_M.gguf b/FastLlama-3.2-1B-Instruct.i1-IQ1_M.gguf new file mode 100644 index 0000000..1423cd6 --- /dev/null +++ b/FastLlama-3.2-1B-Instruct.i1-IQ1_M.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:40f6d43788d6a317b6707e6da1f3a8ef734c2a037d9148846a4be2d4d259d29d +size 413607040 diff --git a/FastLlama-3.2-1B-Instruct.i1-IQ1_S.gguf b/FastLlama-3.2-1B-Instruct.i1-IQ1_S.gguf new file mode 100644 index 0000000..02f5b42 --- /dev/null +++ b/FastLlama-3.2-1B-Instruct.i1-IQ1_S.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:95012c75ba8bb677ad2909e1620605f9a4627726dd795590cfa0aa3208ffbc6f +size 393553024 diff --git a/FastLlama-3.2-1B-Instruct.i1-IQ2_M.gguf b/FastLlama-3.2-1B-Instruct.i1-IQ2_M.gguf new file mode 100644 index 0000000..56e6a91 --- /dev/null +++ b/FastLlama-3.2-1B-Instruct.i1-IQ2_M.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:84d168a7dc47b4bc5507118e7c1335c22755f1da87854f648a266387b39d91e5 +size 515449984 diff --git a/FastLlama-3.2-1B-Instruct.i1-IQ2_S.gguf b/FastLlama-3.2-1B-Instruct.i1-IQ2_S.gguf new file mode 100644 index 0000000..dc80bf3 --- /dev/null +++ b/FastLlama-3.2-1B-Instruct.i1-IQ2_S.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4d796a7df003522dd67ab922095531ec4908935c75879122344a23779f4bb78d +size 488711296 diff --git a/FastLlama-3.2-1B-Instruct.i1-IQ2_XS.gguf b/FastLlama-3.2-1B-Instruct.i1-IQ2_XS.gguf new file mode 100644 index 0000000..bfccee3 --- /dev/null +++ b/FastLlama-3.2-1B-Instruct.i1-IQ2_XS.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:23aeeffd4c0acd156f928014fca4f8cd2104135d8f69b46f188846dba13072fe +size 475866240 diff --git a/FastLlama-3.2-1B-Instruct.i1-IQ2_XXS.gguf b/FastLlama-3.2-1B-Instruct.i1-IQ2_XXS.gguf new file mode 100644 index 0000000..bef91de --- /dev/null +++ b/FastLlama-3.2-1B-Instruct.i1-IQ2_XXS.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d8cd75d79293939e695f9bd2ca5afabe35f99edffc622d9a4ae31587cc2e3da5 +size 447030400 diff --git a/FastLlama-3.2-1B-Instruct.i1-IQ3_M.gguf b/FastLlama-3.2-1B-Instruct.i1-IQ3_M.gguf new file mode 100644 index 0000000..a47a30e --- /dev/null +++ b/FastLlama-3.2-1B-Instruct.i1-IQ3_M.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:91e4d4f917cf55e370da32256b1f9ce2af40bc4e4634337b7788761c0a447585 +size 657290368 diff --git a/FastLlama-3.2-1B-Instruct.i1-IQ3_S.gguf b/FastLlama-3.2-1B-Instruct.i1-IQ3_S.gguf new file mode 100644 index 0000000..8d90c3e --- /dev/null +++ b/FastLlama-3.2-1B-Instruct.i1-IQ3_S.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a078d760b81286ab29984e3d9d9da42f3a2b3e1053e311337cf7613b61ffe511 +size 643921024 diff --git a/FastLlama-3.2-1B-Instruct.i1-IQ3_XS.gguf b/FastLlama-3.2-1B-Instruct.i1-IQ3_XS.gguf new file mode 100644 index 0000000..d81a2f2 --- /dev/null +++ b/FastLlama-3.2-1B-Instruct.i1-IQ3_XS.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7b5fc2e2f1b0180b2d9099fba09efb0b85e20acba16dbb1cfa7f7a61049ccd9c +size 621114496 diff --git a/FastLlama-3.2-1B-Instruct.i1-IQ3_XXS.gguf b/FastLlama-3.2-1B-Instruct.i1-IQ3_XXS.gguf new file mode 100644 index 0000000..b0b60c7 --- /dev/null +++ b/FastLlama-3.2-1B-Instruct.i1-IQ3_XXS.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e8c0c5b191706cd5ff8d25a95fc6308fbc4b83a7b36f282b098ee9f590a52133 +size 562111616 diff --git a/FastLlama-3.2-1B-Instruct.i1-IQ4_XS.gguf b/FastLlama-3.2-1B-Instruct.i1-IQ4_XS.gguf new file mode 100644 index 0000000..e20e70f --- /dev/null +++ b/FastLlama-3.2-1B-Instruct.i1-IQ4_XS.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:39b0fb28c9ff437bac8962999117c3935a4d6592d46a581a7a68198ca9a75fbb +size 743142528 diff --git a/FastLlama-3.2-1B-Instruct.i1-Q2_K.gguf b/FastLlama-3.2-1B-Instruct.i1-Q2_K.gguf new file mode 100644 index 0000000..e437a40 --- /dev/null +++ b/FastLlama-3.2-1B-Instruct.i1-Q2_K.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f89196e677c16d26c240bb5f51c2788b615da8de68d58649f01fd7398846d9d +size 580875392 diff --git a/FastLlama-3.2-1B-Instruct.i1-Q2_K_S.gguf b/FastLlama-3.2-1B-Instruct.i1-Q2_K_S.gguf new file mode 100644 index 0000000..5289fc6 --- /dev/null +++ b/FastLlama-3.2-1B-Instruct.i1-Q2_K_S.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c646c3fabe869978df5cbb2a0c750acd7f5d451a515de58efca4b113f1277f7b +size 554660992 diff --git a/FastLlama-3.2-1B-Instruct.i1-Q3_K_L.gguf b/FastLlama-3.2-1B-Instruct.i1-Q3_K_L.gguf new file mode 100644 index 0000000..c5008da --- /dev/null +++ b/FastLlama-3.2-1B-Instruct.i1-Q3_K_L.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4845de2b93a7c2e975a5abe5bfde35baad8dccb9e640fd06591fc0cace2f65c5 +size 732525696 diff --git a/FastLlama-3.2-1B-Instruct.i1-Q3_K_M.gguf b/FastLlama-3.2-1B-Instruct.i1-Q3_K_M.gguf new file mode 100644 index 0000000..e2fe3c5 --- /dev/null +++ b/FastLlama-3.2-1B-Instruct.i1-Q3_K_M.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a81d0a0d520fbc3e37954d019e5734152272cfadbe01be32dd378eba7b787459 +size 690844800 diff --git a/FastLlama-3.2-1B-Instruct.i1-Q3_K_S.gguf b/FastLlama-3.2-1B-Instruct.i1-Q3_K_S.gguf new file mode 100644 index 0000000..4bbbeb0 --- /dev/null +++ b/FastLlama-3.2-1B-Instruct.i1-Q3_K_S.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0ce5da64382615d52ec41d44eaec037e3b064e1544976d1ac1025e1f4a1a14e8 +size 641692800 diff --git a/FastLlama-3.2-1B-Instruct.i1-Q4_0.gguf b/FastLlama-3.2-1B-Instruct.i1-Q4_0.gguf new file mode 100644 index 0000000..1ca3b3c --- /dev/null +++ b/FastLlama-3.2-1B-Instruct.i1-Q4_0.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:484df12471e1ab76f22a60d6efc9603d40ebedb1b2d197d610bb4545244c7345 +size 773026944 diff --git a/FastLlama-3.2-1B-Instruct.i1-Q4_0_4_4.gguf b/FastLlama-3.2-1B-Instruct.i1-Q4_0_4_4.gguf new file mode 100644 index 0000000..f2131a5 --- /dev/null +++ b/FastLlama-3.2-1B-Instruct.i1-Q4_0_4_4.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7c88c87e6381622df8ac324b2b1d1bc62a76b028eaf6191f126ad6056cb7913d +size 770929792 diff --git a/FastLlama-3.2-1B-Instruct.i1-Q4_0_4_8.gguf b/FastLlama-3.2-1B-Instruct.i1-Q4_0_4_8.gguf new file mode 100644 index 0000000..d290fbc --- /dev/null +++ b/FastLlama-3.2-1B-Instruct.i1-Q4_0_4_8.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:966f964a34c714e727bf70668b77f9a175191eff504fee1965c9d9299f4e2ed9 +size 770929792 diff --git a/FastLlama-3.2-1B-Instruct.i1-Q4_0_8_8.gguf b/FastLlama-3.2-1B-Instruct.i1-Q4_0_8_8.gguf new file mode 100644 index 0000000..cfa6657 --- /dev/null +++ b/FastLlama-3.2-1B-Instruct.i1-Q4_0_8_8.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d970a3f76d4da5125ca9b239f619238f6bcb87232a38f23b4836d9f998da4257 +size 770929792 diff --git a/FastLlama-3.2-1B-Instruct.i1-Q4_K_M.gguf b/FastLlama-3.2-1B-Instruct.i1-Q4_K_M.gguf new file mode 100644 index 0000000..89cc3a3 --- /dev/null +++ b/FastLlama-3.2-1B-Instruct.i1-Q4_K_M.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4ab8a0687ec8e01277b07f3697170f485b45101901c59aa1aba331db6ab86912 +size 807695488 diff --git a/FastLlama-3.2-1B-Instruct.i1-Q4_K_S.gguf b/FastLlama-3.2-1B-Instruct.i1-Q4_K_S.gguf new file mode 100644 index 0000000..1aef9f2 --- /dev/null +++ b/FastLlama-3.2-1B-Instruct.i1-Q4_K_S.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ec878859658151fd6b00695cee8f32354532619014949e8b5bf755d42d134a1 +size 775648384 diff --git a/FastLlama-3.2-1B-Instruct.i1-Q5_K_M.gguf b/FastLlama-3.2-1B-Instruct.i1-Q5_K_M.gguf new file mode 100644 index 0000000..c7620d3 --- /dev/null +++ b/FastLlama-3.2-1B-Instruct.i1-Q5_K_M.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:168a03443701f2f82196e3e07e0f3fb516111da431cf8c84f8e88833ba00e47b +size 911504512 diff --git a/FastLlama-3.2-1B-Instruct.i1-Q5_K_S.gguf b/FastLlama-3.2-1B-Instruct.i1-Q5_K_S.gguf new file mode 100644 index 0000000..909dbf6 --- /dev/null +++ b/FastLlama-3.2-1B-Instruct.i1-Q5_K_S.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6969391806ab42bf6347ab01ecf500d2fa213127ce33fe198243ec0fa1e0103a +size 892564608 diff --git a/FastLlama-3.2-1B-Instruct.i1-Q6_K.gguf b/FastLlama-3.2-1B-Instruct.i1-Q6_K.gguf new file mode 100644 index 0000000..d5217de --- /dev/null +++ b/FastLlama-3.2-1B-Instruct.i1-Q6_K.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1457995b3ed73a6d73002615563a8e1202770a740f495db20f7f10d3018d50d0 +size 1021801600 diff --git a/README.md b/README.md new file mode 100644 index 0000000..0e99cc5 --- /dev/null +++ b/README.md @@ -0,0 +1,96 @@ +--- +base_model: suayptalha/FastLlama-3.2-1B-Instruct +datasets: +- HuggingFaceTB/smoltalk +language: +- en +- de +- es +- fr +- it +- pt +- hi +- th +library_name: transformers +license: apache-2.0 +quantized_by: mradermacher +tags: +- math +- lora +- science +- chemistry +- biology +- code +- text-generation-inference +- unsloth +- llama +--- +## About + + + + + + +weighted/imatrix quants of https://huggingface.co/suayptalha/FastLlama-3.2-1B-Instruct + + +static quants are available at https://huggingface.co/mradermacher/FastLlama-3.2-1B-Instruct-GGUF +## Usage + +If you are unsure how to use GGUF files, refer to one of [TheBloke's +READMEs](https://huggingface.co/TheBloke/KafkaLM-70B-German-V0.1-GGUF) for +more details, including on how to concatenate multi-part files. + +## Provided Quants + +(sorted by size, not necessarily quality. IQ-quants are often preferable over similar sized non-IQ quants) + +| Link | Type | Size/GB | Notes | +|:-----|:-----|--------:|:------| +| [GGUF](https://huggingface.co/mradermacher/FastLlama-3.2-1B-Instruct-i1-GGUF/resolve/main/FastLlama-3.2-1B-Instruct.i1-IQ1_S.gguf) | i1-IQ1_S | 0.5 | for the desperate | +| [GGUF](https://huggingface.co/mradermacher/FastLlama-3.2-1B-Instruct-i1-GGUF/resolve/main/FastLlama-3.2-1B-Instruct.i1-IQ1_M.gguf) | i1-IQ1_M | 0.5 | mostly desperate | +| [GGUF](https://huggingface.co/mradermacher/FastLlama-3.2-1B-Instruct-i1-GGUF/resolve/main/FastLlama-3.2-1B-Instruct.i1-IQ2_XXS.gguf) | i1-IQ2_XXS | 0.5 | | +| [GGUF](https://huggingface.co/mradermacher/FastLlama-3.2-1B-Instruct-i1-GGUF/resolve/main/FastLlama-3.2-1B-Instruct.i1-IQ2_XS.gguf) | i1-IQ2_XS | 0.6 | | +| [GGUF](https://huggingface.co/mradermacher/FastLlama-3.2-1B-Instruct-i1-GGUF/resolve/main/FastLlama-3.2-1B-Instruct.i1-IQ2_S.gguf) | i1-IQ2_S | 0.6 | | +| [GGUF](https://huggingface.co/mradermacher/FastLlama-3.2-1B-Instruct-i1-GGUF/resolve/main/FastLlama-3.2-1B-Instruct.i1-IQ2_M.gguf) | i1-IQ2_M | 0.6 | | +| [GGUF](https://huggingface.co/mradermacher/FastLlama-3.2-1B-Instruct-i1-GGUF/resolve/main/FastLlama-3.2-1B-Instruct.i1-Q2_K_S.gguf) | i1-Q2_K_S | 0.7 | very low quality | +| [GGUF](https://huggingface.co/mradermacher/FastLlama-3.2-1B-Instruct-i1-GGUF/resolve/main/FastLlama-3.2-1B-Instruct.i1-IQ3_XXS.gguf) | i1-IQ3_XXS | 0.7 | lower quality | +| [GGUF](https://huggingface.co/mradermacher/FastLlama-3.2-1B-Instruct-i1-GGUF/resolve/main/FastLlama-3.2-1B-Instruct.i1-Q2_K.gguf) | i1-Q2_K | 0.7 | IQ3_XXS probably better | +| [GGUF](https://huggingface.co/mradermacher/FastLlama-3.2-1B-Instruct-i1-GGUF/resolve/main/FastLlama-3.2-1B-Instruct.i1-IQ3_XS.gguf) | i1-IQ3_XS | 0.7 | | +| [GGUF](https://huggingface.co/mradermacher/FastLlama-3.2-1B-Instruct-i1-GGUF/resolve/main/FastLlama-3.2-1B-Instruct.i1-Q3_K_S.gguf) | i1-Q3_K_S | 0.7 | IQ3_XS probably better | +| [GGUF](https://huggingface.co/mradermacher/FastLlama-3.2-1B-Instruct-i1-GGUF/resolve/main/FastLlama-3.2-1B-Instruct.i1-IQ3_S.gguf) | i1-IQ3_S | 0.7 | beats Q3_K* | +| [GGUF](https://huggingface.co/mradermacher/FastLlama-3.2-1B-Instruct-i1-GGUF/resolve/main/FastLlama-3.2-1B-Instruct.i1-IQ3_M.gguf) | i1-IQ3_M | 0.8 | | +| [GGUF](https://huggingface.co/mradermacher/FastLlama-3.2-1B-Instruct-i1-GGUF/resolve/main/FastLlama-3.2-1B-Instruct.i1-Q3_K_M.gguf) | i1-Q3_K_M | 0.8 | IQ3_S probably better | +| [GGUF](https://huggingface.co/mradermacher/FastLlama-3.2-1B-Instruct-i1-GGUF/resolve/main/FastLlama-3.2-1B-Instruct.i1-Q3_K_L.gguf) | i1-Q3_K_L | 0.8 | IQ3_M probably better | +| [GGUF](https://huggingface.co/mradermacher/FastLlama-3.2-1B-Instruct-i1-GGUF/resolve/main/FastLlama-3.2-1B-Instruct.i1-IQ4_XS.gguf) | i1-IQ4_XS | 0.8 | | +| [GGUF](https://huggingface.co/mradermacher/FastLlama-3.2-1B-Instruct-i1-GGUF/resolve/main/FastLlama-3.2-1B-Instruct.i1-Q4_0_4_4.gguf) | i1-Q4_0_4_4 | 0.9 | fast on arm, low quality | +| [GGUF](https://huggingface.co/mradermacher/FastLlama-3.2-1B-Instruct-i1-GGUF/resolve/main/FastLlama-3.2-1B-Instruct.i1-Q4_0_4_8.gguf) | i1-Q4_0_4_8 | 0.9 | fast on arm+i8mm, low quality | +| [GGUF](https://huggingface.co/mradermacher/FastLlama-3.2-1B-Instruct-i1-GGUF/resolve/main/FastLlama-3.2-1B-Instruct.i1-Q4_0_8_8.gguf) | i1-Q4_0_8_8 | 0.9 | fast on arm+sve, low quality | +| [GGUF](https://huggingface.co/mradermacher/FastLlama-3.2-1B-Instruct-i1-GGUF/resolve/main/FastLlama-3.2-1B-Instruct.i1-Q4_0.gguf) | i1-Q4_0 | 0.9 | fast, low quality | +| [GGUF](https://huggingface.co/mradermacher/FastLlama-3.2-1B-Instruct-i1-GGUF/resolve/main/FastLlama-3.2-1B-Instruct.i1-Q4_K_S.gguf) | i1-Q4_K_S | 0.9 | optimal size/speed/quality | +| [GGUF](https://huggingface.co/mradermacher/FastLlama-3.2-1B-Instruct-i1-GGUF/resolve/main/FastLlama-3.2-1B-Instruct.i1-Q4_K_M.gguf) | i1-Q4_K_M | 0.9 | fast, recommended | +| [GGUF](https://huggingface.co/mradermacher/FastLlama-3.2-1B-Instruct-i1-GGUF/resolve/main/FastLlama-3.2-1B-Instruct.i1-Q5_K_S.gguf) | i1-Q5_K_S | 1.0 | | +| [GGUF](https://huggingface.co/mradermacher/FastLlama-3.2-1B-Instruct-i1-GGUF/resolve/main/FastLlama-3.2-1B-Instruct.i1-Q5_K_M.gguf) | i1-Q5_K_M | 1.0 | | +| [GGUF](https://huggingface.co/mradermacher/FastLlama-3.2-1B-Instruct-i1-GGUF/resolve/main/FastLlama-3.2-1B-Instruct.i1-Q6_K.gguf) | i1-Q6_K | 1.1 | practically like static Q6_K | + +Here is a handy graph by ikawrakow comparing some lower-quality quant +types (lower is better): + +![image.png](https://www.nethype.de/huggingface_embed/quantpplgraph.png) + +And here are Artefact2's thoughts on the matter: +https://gist.github.com/Artefact2/b5f810600771265fc1e39442288e8ec9 + +## FAQ / Model Request + +See https://huggingface.co/mradermacher/model_requests for some answers to +questions you might have and/or if you want some other model quantized. + +## Thanks + +I thank my company, [nethype GmbH](https://www.nethype.de/), for letting +me use its servers and providing upgrades to my workstation to enable +this work in my free time. Additional thanks to [@nicoboss](https://huggingface.co/nicoboss) for giving me access to his private supercomputer, enabling me to provide many more imatrix quants, at much higher quality, than I would otherwise be able to. + + diff --git a/imatrix.dat b/imatrix.dat new file mode 100644 index 0000000..ba1fcc6 --- /dev/null +++ b/imatrix.dat @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c868620ab6f13e92a28397ba40efd519bab59d807109b3a9477486d0c50dfb7a +size 1314413