commit cc4dd0ff0645582bc8b1ed853644ee1c13b9b557 Author: ModelHub XC Date: Mon Sep 28 16:31:17 2026 +0800 初始化项目，由ModelHub XC社区提供模型 Model: mradermacher/Llama-3.1-Swallow-8B-v0.2-i1-GGUF Source: Original Platform diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..81fa879 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,60 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +imatrix.dat filter=lfs diff=lfs merge=lfs -text +Llama-3.1-Swallow-8B-v0.2.i1-Q2_K.gguf filter=lfs diff=lfs merge=lfs -text +Llama-3.1-Swallow-8B-v0.2.i1-IQ3_M.gguf filter=lfs diff=lfs merge=lfs -text +Llama-3.1-Swallow-8B-v0.2.i1-Q4_K_S.gguf filter=lfs diff=lfs merge=lfs -text +Llama-3.1-Swallow-8B-v0.2.i1-IQ3_XXS.gguf filter=lfs diff=lfs merge=lfs -text +Llama-3.1-Swallow-8B-v0.2.i1-Q3_K_M.gguf filter=lfs diff=lfs merge=lfs -text +Llama-3.1-Swallow-8B-v0.2.i1-Q4_K_M.gguf filter=lfs diff=lfs merge=lfs -text +Llama-3.1-Swallow-8B-v0.2.i1-IQ2_M.gguf filter=lfs diff=lfs merge=lfs -text +Llama-3.1-Swallow-8B-v0.2.i1-Q6_K.gguf filter=lfs diff=lfs merge=lfs -text +Llama-3.1-Swallow-8B-v0.2.i1-IQ1_M.gguf filter=lfs diff=lfs merge=lfs -text +Llama-3.1-Swallow-8B-v0.2.i1-IQ4_XS.gguf filter=lfs diff=lfs merge=lfs -text +Llama-3.1-Swallow-8B-v0.2.i1-Q3_K_S.gguf filter=lfs diff=lfs merge=lfs -text +Llama-3.1-Swallow-8B-v0.2.i1-IQ2_XXS.gguf filter=lfs diff=lfs merge=lfs -text +Llama-3.1-Swallow-8B-v0.2.i1-Q3_K_L.gguf filter=lfs diff=lfs merge=lfs -text +Llama-3.1-Swallow-8B-v0.2.i1-IQ2_XS.gguf filter=lfs diff=lfs merge=lfs -text +Llama-3.1-Swallow-8B-v0.2.i1-Q5_K_S.gguf filter=lfs diff=lfs merge=lfs -text +Llama-3.1-Swallow-8B-v0.2.i1-IQ2_S.gguf filter=lfs diff=lfs merge=lfs -text +Llama-3.1-Swallow-8B-v0.2.i1-Q4_0_4_4.gguf filter=lfs diff=lfs merge=lfs -text +Llama-3.1-Swallow-8B-v0.2.i1-IQ1_S.gguf filter=lfs diff=lfs merge=lfs -text +Llama-3.1-Swallow-8B-v0.2.i1-Q5_K_M.gguf filter=lfs diff=lfs merge=lfs -text +Llama-3.1-Swallow-8B-v0.2.i1-Q4_0.gguf filter=lfs diff=lfs merge=lfs -text +Llama-3.1-Swallow-8B-v0.2.i1-IQ3_XS.gguf filter=lfs diff=lfs merge=lfs -text +Llama-3.1-Swallow-8B-v0.2.i1-Q4_0_4_8.gguf filter=lfs diff=lfs merge=lfs -text +Llama-3.1-Swallow-8B-v0.2.i1-Q4_0_8_8.gguf filter=lfs diff=lfs merge=lfs -text +Llama-3.1-Swallow-8B-v0.2.i1-IQ3_S.gguf filter=lfs diff=lfs merge=lfs -text diff --git a/Llama-3.1-Swallow-8B-v0.2.i1-IQ1_M.gguf b/Llama-3.1-Swallow-8B-v0.2.i1-IQ1_M.gguf new file mode 100644 index 0000000..775f065 --- /dev/null +++ b/Llama-3.1-Swallow-8B-v0.2.i1-IQ1_M.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6726723d728d36889f27f51d3199c3efda1c219b937687561dc4ea5bdded76ce +size 2161972480 diff --git a/Llama-3.1-Swallow-8B-v0.2.i1-IQ1_S.gguf b/Llama-3.1-Swallow-8B-v0.2.i1-IQ1_S.gguf new file mode 100644 index 0000000..8c138fd --- /dev/null +++ b/Llama-3.1-Swallow-8B-v0.2.i1-IQ1_S.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7a83908d1fba970fc3009b31c8c587d1edf6be15d73f0489f81760a5b315e07c +size 2019628288 diff --git a/Llama-3.1-Swallow-8B-v0.2.i1-IQ2_M.gguf b/Llama-3.1-Swallow-8B-v0.2.i1-IQ2_M.gguf new file mode 100644 index 0000000..66e3d9a --- /dev/null +++ b/Llama-3.1-Swallow-8B-v0.2.i1-IQ2_M.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e0065b1edd5621c7b51996482cf9e4646d51475d85e440d0540731b967e20361 +size 2948281600 diff --git a/Llama-3.1-Swallow-8B-v0.2.i1-IQ2_S.gguf b/Llama-3.1-Swallow-8B-v0.2.i1-IQ2_S.gguf new file mode 100644 index 0000000..67e7ee8 --- /dev/null +++ b/Llama-3.1-Swallow-8B-v0.2.i1-IQ2_S.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e377fc46b4875a7d812b6ee13a68d33e07b50c0c3f296fb611aebec8fec3e6b1 +size 2758489344 diff --git a/Llama-3.1-Swallow-8B-v0.2.i1-IQ2_XS.gguf b/Llama-3.1-Swallow-8B-v0.2.i1-IQ2_XS.gguf new file mode 100644 index 0000000..fe00b62 --- /dev/null +++ b/Llama-3.1-Swallow-8B-v0.2.i1-IQ2_XS.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f60624d226916d751e2de814aaa61d0ea39eb5089ffce329944bd9103177e33c +size 2605782272 diff --git a/Llama-3.1-Swallow-8B-v0.2.i1-IQ2_XXS.gguf b/Llama-3.1-Swallow-8B-v0.2.i1-IQ2_XXS.gguf new file mode 100644 index 0000000..24876cf --- /dev/null +++ b/Llama-3.1-Swallow-8B-v0.2.i1-IQ2_XXS.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3428a3d0363190a7e11f95754d3c53b940cde3ff70270c8e1367219af924bfde +size 2399212800 diff --git a/Llama-3.1-Swallow-8B-v0.2.i1-IQ3_M.gguf b/Llama-3.1-Swallow-8B-v0.2.i1-IQ3_M.gguf new file mode 100644 index 0000000..9ddc0b4 --- /dev/null +++ b/Llama-3.1-Swallow-8B-v0.2.i1-IQ3_M.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b27be39208ec72ff88302fd295f97dc876817792ec66cbc500f8d2182f391a8b +size 3784824064 diff --git a/Llama-3.1-Swallow-8B-v0.2.i1-IQ3_S.gguf b/Llama-3.1-Swallow-8B-v0.2.i1-IQ3_S.gguf new file mode 100644 index 0000000..852caa6 --- /dev/null +++ b/Llama-3.1-Swallow-8B-v0.2.i1-IQ3_S.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bf15c17cfb9708f6be4a11e0df0a3c3a1647d0a685a8f4c6bc3ea02c9e6b8f9a +size 3682325760 diff --git a/Llama-3.1-Swallow-8B-v0.2.i1-IQ3_XS.gguf b/Llama-3.1-Swallow-8B-v0.2.i1-IQ3_XS.gguf new file mode 100644 index 0000000..bcb7e12 --- /dev/null +++ b/Llama-3.1-Swallow-8B-v0.2.i1-IQ3_XS.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4fd0d0a0eebf23cf63e3672f323e393dc8336036339693b9b6309eb842e37fe9 +size 3518747904 diff --git a/Llama-3.1-Swallow-8B-v0.2.i1-IQ3_XXS.gguf b/Llama-3.1-Swallow-8B-v0.2.i1-IQ3_XXS.gguf new file mode 100644 index 0000000..5975687 --- /dev/null +++ b/Llama-3.1-Swallow-8B-v0.2.i1-IQ3_XXS.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f2bcc778b6d7b3fce88247decbfc806784a09bb2d59566c71df48f2da39d409f +size 3274913024 diff --git a/Llama-3.1-Swallow-8B-v0.2.i1-IQ4_XS.gguf b/Llama-3.1-Swallow-8B-v0.2.i1-IQ4_XS.gguf new file mode 100644 index 0000000..1793fdc --- /dev/null +++ b/Llama-3.1-Swallow-8B-v0.2.i1-IQ4_XS.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3612c7a8d9d548e8fe6cc9b9de069c20e0afaae30186ad54511468d656b0d6bd +size 4447663360 diff --git a/Llama-3.1-Swallow-8B-v0.2.i1-Q2_K.gguf b/Llama-3.1-Swallow-8B-v0.2.i1-Q2_K.gguf new file mode 100644 index 0000000..790737b --- /dev/null +++ b/Llama-3.1-Swallow-8B-v0.2.i1-Q2_K.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f8a55ffd98e82fe4cd5fc875f934ffa5b5b70c2567c01738eaeb1ae54f4d735e +size 3179132160 diff --git a/Llama-3.1-Swallow-8B-v0.2.i1-Q3_K_L.gguf b/Llama-3.1-Swallow-8B-v0.2.i1-Q3_K_L.gguf new file mode 100644 index 0000000..cc25bd9 --- /dev/null +++ b/Llama-3.1-Swallow-8B-v0.2.i1-Q3_K_L.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:294f1f4d1faeaeb5818bd7af8dc360446fad3aade5fca20cfd98c94740957a6e +size 4321957120 diff --git a/Llama-3.1-Swallow-8B-v0.2.i1-Q3_K_M.gguf b/Llama-3.1-Swallow-8B-v0.2.i1-Q3_K_M.gguf new file mode 100644 index 0000000..d8fd4f6 --- /dev/null +++ b/Llama-3.1-Swallow-8B-v0.2.i1-Q3_K_M.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:75dcf1c5c3288c2f3046e2c67b8918664b2f0f925617e487d5ed2ab83bfcc2f4 +size 4018918656 diff --git a/Llama-3.1-Swallow-8B-v0.2.i1-Q3_K_S.gguf b/Llama-3.1-Swallow-8B-v0.2.i1-Q3_K_S.gguf new file mode 100644 index 0000000..c41a759 --- /dev/null +++ b/Llama-3.1-Swallow-8B-v0.2.i1-Q3_K_S.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:22d0ba9c773178f78c7d6d20f9de6d96af33c6abd9dbfcd943e195b7bca1d991 +size 3664499968 diff --git a/Llama-3.1-Swallow-8B-v0.2.i1-Q4_0.gguf b/Llama-3.1-Swallow-8B-v0.2.i1-Q4_0.gguf new file mode 100644 index 0000000..6ee2631 --- /dev/null +++ b/Llama-3.1-Swallow-8B-v0.2.i1-Q4_0.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9c69d66f491546c64a36471da61e6ba11e0320199b23772ca423696598bf15a3 +size 4675892480 diff --git a/Llama-3.1-Swallow-8B-v0.2.i1-Q4_0_4_4.gguf b/Llama-3.1-Swallow-8B-v0.2.i1-Q4_0_4_4.gguf new file mode 100644 index 0000000..841df4e --- /dev/null +++ b/Llama-3.1-Swallow-8B-v0.2.i1-Q4_0_4_4.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:96a04856aba5b2d0f56b4cffebcc3fa94f18799742cf71bd8f0f67575caa5279 +size 4661212416 diff --git a/Llama-3.1-Swallow-8B-v0.2.i1-Q4_0_4_8.gguf b/Llama-3.1-Swallow-8B-v0.2.i1-Q4_0_4_8.gguf new file mode 100644 index 0000000..15e66f3 --- /dev/null +++ b/Llama-3.1-Swallow-8B-v0.2.i1-Q4_0_4_8.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:219af55fe88c5d6c0d21d72f76d605344a6ed4a5295ec682b5b39edf4a3484de +size 4661212416 diff --git a/Llama-3.1-Swallow-8B-v0.2.i1-Q4_0_8_8.gguf b/Llama-3.1-Swallow-8B-v0.2.i1-Q4_0_8_8.gguf new file mode 100644 index 0000000..934c1c9 --- /dev/null +++ b/Llama-3.1-Swallow-8B-v0.2.i1-Q4_0_8_8.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f118cc9e9663660fe345e06867321a9194cd6484ad54ccec2178793d73dd293d +size 4661212416 diff --git a/Llama-3.1-Swallow-8B-v0.2.i1-Q4_K_M.gguf b/Llama-3.1-Swallow-8B-v0.2.i1-Q4_K_M.gguf new file mode 100644 index 0000000..7b56398 --- /dev/null +++ b/Llama-3.1-Swallow-8B-v0.2.i1-Q4_K_M.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:821201e08bcd8c44d67d036b002c2f6054281dd1c3c35f7438571e68c2784663 +size 4920734976 diff --git a/Llama-3.1-Swallow-8B-v0.2.i1-Q4_K_S.gguf b/Llama-3.1-Swallow-8B-v0.2.i1-Q4_K_S.gguf new file mode 100644 index 0000000..48cea6f --- /dev/null +++ b/Llama-3.1-Swallow-8B-v0.2.i1-Q4_K_S.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:189e36895d6c5a66d1a73672b69e5295e76954bffd0db57499737663a637987f +size 4692669696 diff --git a/Llama-3.1-Swallow-8B-v0.2.i1-Q5_K_M.gguf b/Llama-3.1-Swallow-8B-v0.2.i1-Q5_K_M.gguf new file mode 100644 index 0000000..2c269bf --- /dev/null +++ b/Llama-3.1-Swallow-8B-v0.2.i1-Q5_K_M.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7ed62178c317813b392f5c956408a563b63786f895bd7b73d34c10071d1e925c +size 5732988160 diff --git a/Llama-3.1-Swallow-8B-v0.2.i1-Q5_K_S.gguf b/Llama-3.1-Swallow-8B-v0.2.i1-Q5_K_S.gguf new file mode 100644 index 0000000..9516601 --- /dev/null +++ b/Llama-3.1-Swallow-8B-v0.2.i1-Q5_K_S.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:07c8e0c979ba6168aaafd9b5d10ec0856fa20b1cf1d35e5bc6e235f144bccd86 +size 5599294720 diff --git a/Llama-3.1-Swallow-8B-v0.2.i1-Q6_K.gguf b/Llama-3.1-Swallow-8B-v0.2.i1-Q6_K.gguf new file mode 100644 index 0000000..b913ee4 --- /dev/null +++ b/Llama-3.1-Swallow-8B-v0.2.i1-Q6_K.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9c2f05649303f184c9aaae309601390d1fda45164dbdb7670796a33f51d9842b +size 6596007168 diff --git a/README.md b/README.md new file mode 100644 index 0000000..bcbd8cf --- /dev/null +++ b/README.md @@ -0,0 +1,78 @@ +--- +base_model: tokyotech-llm/Llama-3.1-Swallow-8B-v0.2 +language: +- en +- ja +library_name: transformers +license: llama3.1 +model_type: llama +quantized_by: mradermacher +--- +## About + + + + + + +weighted/imatrix quants of https://huggingface.co/tokyotech-llm/Llama-3.1-Swallow-8B-v0.2 + + +static quants are available at https://huggingface.co/mradermacher/Llama-3.1-Swallow-8B-v0.2-GGUF +## Usage + +If you are unsure how to use GGUF files, refer to one of [TheBloke's +READMEs](https://huggingface.co/TheBloke/KafkaLM-70B-German-V0.1-GGUF) for +more details, including on how to concatenate multi-part files. + +## Provided Quants + +(sorted by size, not necessarily quality. IQ-quants are often preferable over similar sized non-IQ quants) + +| Link | Type | Size/GB | Notes | +|:-----|:-----|--------:|:------| +| [GGUF](https://huggingface.co/mradermacher/Llama-3.1-Swallow-8B-v0.2-i1-GGUF/resolve/main/Llama-3.1-Swallow-8B-v0.2.i1-IQ1_S.gguf) | i1-IQ1_S | 2.1 | for the desperate | +| [GGUF](https://huggingface.co/mradermacher/Llama-3.1-Swallow-8B-v0.2-i1-GGUF/resolve/main/Llama-3.1-Swallow-8B-v0.2.i1-IQ1_M.gguf) | i1-IQ1_M | 2.3 | mostly desperate | +| [GGUF](https://huggingface.co/mradermacher/Llama-3.1-Swallow-8B-v0.2-i1-GGUF/resolve/main/Llama-3.1-Swallow-8B-v0.2.i1-IQ2_XXS.gguf) | i1-IQ2_XXS | 2.5 | | +| [GGUF](https://huggingface.co/mradermacher/Llama-3.1-Swallow-8B-v0.2-i1-GGUF/resolve/main/Llama-3.1-Swallow-8B-v0.2.i1-IQ2_XS.gguf) | i1-IQ2_XS | 2.7 | | +| [GGUF](https://huggingface.co/mradermacher/Llama-3.1-Swallow-8B-v0.2-i1-GGUF/resolve/main/Llama-3.1-Swallow-8B-v0.2.i1-IQ2_S.gguf) | i1-IQ2_S | 2.9 | | +| [GGUF](https://huggingface.co/mradermacher/Llama-3.1-Swallow-8B-v0.2-i1-GGUF/resolve/main/Llama-3.1-Swallow-8B-v0.2.i1-IQ2_M.gguf) | i1-IQ2_M | 3.0 | | +| [GGUF](https://huggingface.co/mradermacher/Llama-3.1-Swallow-8B-v0.2-i1-GGUF/resolve/main/Llama-3.1-Swallow-8B-v0.2.i1-Q2_K.gguf) | i1-Q2_K | 3.3 | IQ3_XXS probably better | +| [GGUF](https://huggingface.co/mradermacher/Llama-3.1-Swallow-8B-v0.2-i1-GGUF/resolve/main/Llama-3.1-Swallow-8B-v0.2.i1-IQ3_XXS.gguf) | i1-IQ3_XXS | 3.4 | lower quality | +| [GGUF](https://huggingface.co/mradermacher/Llama-3.1-Swallow-8B-v0.2-i1-GGUF/resolve/main/Llama-3.1-Swallow-8B-v0.2.i1-IQ3_XS.gguf) | i1-IQ3_XS | 3.6 | | +| [GGUF](https://huggingface.co/mradermacher/Llama-3.1-Swallow-8B-v0.2-i1-GGUF/resolve/main/Llama-3.1-Swallow-8B-v0.2.i1-Q3_K_S.gguf) | i1-Q3_K_S | 3.8 | IQ3_XS probably better | +| [GGUF](https://huggingface.co/mradermacher/Llama-3.1-Swallow-8B-v0.2-i1-GGUF/resolve/main/Llama-3.1-Swallow-8B-v0.2.i1-IQ3_S.gguf) | i1-IQ3_S | 3.8 | beats Q3_K* | +| [GGUF](https://huggingface.co/mradermacher/Llama-3.1-Swallow-8B-v0.2-i1-GGUF/resolve/main/Llama-3.1-Swallow-8B-v0.2.i1-IQ3_M.gguf) | i1-IQ3_M | 3.9 | | +| [GGUF](https://huggingface.co/mradermacher/Llama-3.1-Swallow-8B-v0.2-i1-GGUF/resolve/main/Llama-3.1-Swallow-8B-v0.2.i1-Q3_K_M.gguf) | i1-Q3_K_M | 4.1 | IQ3_S probably better | +| [GGUF](https://huggingface.co/mradermacher/Llama-3.1-Swallow-8B-v0.2-i1-GGUF/resolve/main/Llama-3.1-Swallow-8B-v0.2.i1-Q3_K_L.gguf) | i1-Q3_K_L | 4.4 | IQ3_M probably better | +| [GGUF](https://huggingface.co/mradermacher/Llama-3.1-Swallow-8B-v0.2-i1-GGUF/resolve/main/Llama-3.1-Swallow-8B-v0.2.i1-IQ4_XS.gguf) | i1-IQ4_XS | 4.5 | | +| [GGUF](https://huggingface.co/mradermacher/Llama-3.1-Swallow-8B-v0.2-i1-GGUF/resolve/main/Llama-3.1-Swallow-8B-v0.2.i1-Q4_0_4_4.gguf) | i1-Q4_0_4_4 | 4.8 | fast on arm, low quality | +| [GGUF](https://huggingface.co/mradermacher/Llama-3.1-Swallow-8B-v0.2-i1-GGUF/resolve/main/Llama-3.1-Swallow-8B-v0.2.i1-Q4_0_4_8.gguf) | i1-Q4_0_4_8 | 4.8 | fast on arm+i8mm, low quality | +| [GGUF](https://huggingface.co/mradermacher/Llama-3.1-Swallow-8B-v0.2-i1-GGUF/resolve/main/Llama-3.1-Swallow-8B-v0.2.i1-Q4_0_8_8.gguf) | i1-Q4_0_8_8 | 4.8 | fast on arm+sve, low quality | +| [GGUF](https://huggingface.co/mradermacher/Llama-3.1-Swallow-8B-v0.2-i1-GGUF/resolve/main/Llama-3.1-Swallow-8B-v0.2.i1-Q4_0.gguf) | i1-Q4_0 | 4.8 | fast, low quality | +| [GGUF](https://huggingface.co/mradermacher/Llama-3.1-Swallow-8B-v0.2-i1-GGUF/resolve/main/Llama-3.1-Swallow-8B-v0.2.i1-Q4_K_S.gguf) | i1-Q4_K_S | 4.8 | optimal size/speed/quality | +| [GGUF](https://huggingface.co/mradermacher/Llama-3.1-Swallow-8B-v0.2-i1-GGUF/resolve/main/Llama-3.1-Swallow-8B-v0.2.i1-Q4_K_M.gguf) | i1-Q4_K_M | 5.0 | fast, recommended | +| [GGUF](https://huggingface.co/mradermacher/Llama-3.1-Swallow-8B-v0.2-i1-GGUF/resolve/main/Llama-3.1-Swallow-8B-v0.2.i1-Q5_K_S.gguf) | i1-Q5_K_S | 5.7 | | +| [GGUF](https://huggingface.co/mradermacher/Llama-3.1-Swallow-8B-v0.2-i1-GGUF/resolve/main/Llama-3.1-Swallow-8B-v0.2.i1-Q5_K_M.gguf) | i1-Q5_K_M | 5.8 | | +| [GGUF](https://huggingface.co/mradermacher/Llama-3.1-Swallow-8B-v0.2-i1-GGUF/resolve/main/Llama-3.1-Swallow-8B-v0.2.i1-Q6_K.gguf) | i1-Q6_K | 6.7 | practically like static Q6_K | + +Here is a handy graph by ikawrakow comparing some lower-quality quant +types (lower is better): + +![image.png](https://www.nethype.de/huggingface_embed/quantpplgraph.png) + +And here are Artefact2's thoughts on the matter: +https://gist.github.com/Artefact2/b5f810600771265fc1e39442288e8ec9 + +## FAQ / Model Request + +See https://huggingface.co/mradermacher/model_requests for some answers to +questions you might have and/or if you want some other model quantized. + +## Thanks + +I thank my company, [nethype GmbH](https://www.nethype.de/), for letting +me use its servers and providing upgrades to my workstation to enable +this work in my free time. Additional thanks to [@nicoboss](https://huggingface.co/nicoboss) for giving me access to his private supercomputer, enabling me to provide many more imatrix quants, at much higher quality, than I would otherwise be able to. + + diff --git a/imatrix.dat b/imatrix.dat new file mode 100644 index 0000000..5b3e163 --- /dev/null +++ b/imatrix.dat @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:de71782bf19af39c93c8060c3c06c75561f6546a1d006c44670a9eeaa066992d +size 4988157