From 82871e6bd25698f891040bb55023e88bee6efe07 Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Mon, 4 May 2026 17:35:00 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: mradermacher/Mistral-quiet-star-demo-i1-GGUF Source: Original Platform --- .gitattributes | 60 +++++++++++++++++ Mistral-quiet-star-demo.i1-IQ1_M.gguf | 3 + Mistral-quiet-star-demo.i1-IQ1_S.gguf | 3 + Mistral-quiet-star-demo.i1-IQ2_M.gguf | 3 + Mistral-quiet-star-demo.i1-IQ2_S.gguf | 3 + Mistral-quiet-star-demo.i1-IQ2_XS.gguf | 3 + Mistral-quiet-star-demo.i1-IQ2_XXS.gguf | 3 + Mistral-quiet-star-demo.i1-IQ3_M.gguf | 3 + Mistral-quiet-star-demo.i1-IQ3_S.gguf | 3 + Mistral-quiet-star-demo.i1-IQ3_XS.gguf | 3 + Mistral-quiet-star-demo.i1-IQ3_XXS.gguf | 3 + Mistral-quiet-star-demo.i1-IQ4_XS.gguf | 3 + Mistral-quiet-star-demo.i1-Q2_K.gguf | 3 + Mistral-quiet-star-demo.i1-Q3_K_L.gguf | 3 + Mistral-quiet-star-demo.i1-Q3_K_M.gguf | 3 + Mistral-quiet-star-demo.i1-Q3_K_S.gguf | 3 + Mistral-quiet-star-demo.i1-Q4_0.gguf | 3 + Mistral-quiet-star-demo.i1-Q4_0_4_4.gguf | 3 + Mistral-quiet-star-demo.i1-Q4_0_4_8.gguf | 3 + Mistral-quiet-star-demo.i1-Q4_0_8_8.gguf | 3 + Mistral-quiet-star-demo.i1-Q4_K_M.gguf | 3 + Mistral-quiet-star-demo.i1-Q4_K_S.gguf | 3 + Mistral-quiet-star-demo.i1-Q5_K_M.gguf | 3 + Mistral-quiet-star-demo.i1-Q5_K_S.gguf | 3 + Mistral-quiet-star-demo.i1-Q6_K.gguf | 3 + README.md | 84 ++++++++++++++++++++++++ imatrix.dat | 3 + 27 files changed, 219 insertions(+) create mode 100644 .gitattributes create mode 100644 Mistral-quiet-star-demo.i1-IQ1_M.gguf create mode 100644 Mistral-quiet-star-demo.i1-IQ1_S.gguf create mode 100644 Mistral-quiet-star-demo.i1-IQ2_M.gguf create mode 100644 Mistral-quiet-star-demo.i1-IQ2_S.gguf create mode 100644 Mistral-quiet-star-demo.i1-IQ2_XS.gguf create mode 100644 Mistral-quiet-star-demo.i1-IQ2_XXS.gguf create mode 100644 Mistral-quiet-star-demo.i1-IQ3_M.gguf create mode 100644 Mistral-quiet-star-demo.i1-IQ3_S.gguf create mode 100644 Mistral-quiet-star-demo.i1-IQ3_XS.gguf create mode 100644 Mistral-quiet-star-demo.i1-IQ3_XXS.gguf create mode 100644 Mistral-quiet-star-demo.i1-IQ4_XS.gguf create mode 100644 Mistral-quiet-star-demo.i1-Q2_K.gguf create mode 100644 Mistral-quiet-star-demo.i1-Q3_K_L.gguf create mode 100644 Mistral-quiet-star-demo.i1-Q3_K_M.gguf create mode 100644 Mistral-quiet-star-demo.i1-Q3_K_S.gguf create mode 100644 Mistral-quiet-star-demo.i1-Q4_0.gguf create mode 100644 Mistral-quiet-star-demo.i1-Q4_0_4_4.gguf create mode 100644 Mistral-quiet-star-demo.i1-Q4_0_4_8.gguf create mode 100644 Mistral-quiet-star-demo.i1-Q4_0_8_8.gguf create mode 100644 Mistral-quiet-star-demo.i1-Q4_K_M.gguf create mode 100644 Mistral-quiet-star-demo.i1-Q4_K_S.gguf create mode 100644 Mistral-quiet-star-demo.i1-Q5_K_M.gguf create mode 100644 Mistral-quiet-star-demo.i1-Q5_K_S.gguf create mode 100644 Mistral-quiet-star-demo.i1-Q6_K.gguf create mode 100644 README.md create mode 100644 imatrix.dat diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..0ceb1c4 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,60 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +imatrix.dat filter=lfs diff=lfs merge=lfs -text +Mistral-quiet-star-demo.i1-Q2_K.gguf filter=lfs diff=lfs merge=lfs -text +Mistral-quiet-star-demo.i1-IQ3_M.gguf filter=lfs diff=lfs merge=lfs -text +Mistral-quiet-star-demo.i1-Q4_K_S.gguf filter=lfs diff=lfs merge=lfs -text +Mistral-quiet-star-demo.i1-IQ3_XXS.gguf filter=lfs diff=lfs merge=lfs -text +Mistral-quiet-star-demo.i1-Q4_K_M.gguf filter=lfs diff=lfs merge=lfs -text +Mistral-quiet-star-demo.i1-IQ2_M.gguf filter=lfs diff=lfs merge=lfs -text +Mistral-quiet-star-demo.i1-Q3_K_M.gguf filter=lfs diff=lfs merge=lfs -text +Mistral-quiet-star-demo.i1-Q6_K.gguf filter=lfs diff=lfs merge=lfs -text +Mistral-quiet-star-demo.i1-IQ1_M.gguf filter=lfs diff=lfs merge=lfs -text +Mistral-quiet-star-demo.i1-IQ4_XS.gguf filter=lfs diff=lfs merge=lfs -text +Mistral-quiet-star-demo.i1-Q3_K_S.gguf filter=lfs diff=lfs merge=lfs -text +Mistral-quiet-star-demo.i1-IQ2_XXS.gguf filter=lfs diff=lfs merge=lfs -text +Mistral-quiet-star-demo.i1-Q3_K_L.gguf filter=lfs diff=lfs merge=lfs -text +Mistral-quiet-star-demo.i1-IQ2_XS.gguf filter=lfs diff=lfs merge=lfs -text +Mistral-quiet-star-demo.i1-Q5_K_S.gguf filter=lfs diff=lfs merge=lfs -text +Mistral-quiet-star-demo.i1-IQ2_S.gguf filter=lfs diff=lfs merge=lfs -text +Mistral-quiet-star-demo.i1-Q4_0_4_4.gguf filter=lfs diff=lfs merge=lfs -text +Mistral-quiet-star-demo.i1-IQ1_S.gguf filter=lfs diff=lfs merge=lfs -text +Mistral-quiet-star-demo.i1-Q5_K_M.gguf filter=lfs diff=lfs merge=lfs -text +Mistral-quiet-star-demo.i1-Q4_0.gguf filter=lfs diff=lfs merge=lfs -text +Mistral-quiet-star-demo.i1-IQ3_XS.gguf filter=lfs diff=lfs merge=lfs -text +Mistral-quiet-star-demo.i1-Q4_0_4_8.gguf filter=lfs diff=lfs merge=lfs -text +Mistral-quiet-star-demo.i1-Q4_0_8_8.gguf filter=lfs diff=lfs merge=lfs -text +Mistral-quiet-star-demo.i1-IQ3_S.gguf filter=lfs diff=lfs merge=lfs -text diff --git a/Mistral-quiet-star-demo.i1-IQ1_M.gguf b/Mistral-quiet-star-demo.i1-IQ1_M.gguf new file mode 100644 index 0000000..e9a3ad6 --- /dev/null +++ b/Mistral-quiet-star-demo.i1-IQ1_M.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:531ee253ced8098831756353b155cfc978846361b7fd8b105ba295e6cdf7cbdd +size 1754447136 diff --git a/Mistral-quiet-star-demo.i1-IQ1_S.gguf b/Mistral-quiet-star-demo.i1-IQ1_S.gguf new file mode 100644 index 0000000..70dbed3 --- /dev/null +++ b/Mistral-quiet-star-demo.i1-IQ1_S.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1907a6475894ec1bd9fd2ca7340d2ad1c410efdf2c3ae9769e94e5195019f3e7 +size 1612102944 diff --git a/Mistral-quiet-star-demo.i1-IQ2_M.gguf b/Mistral-quiet-star-demo.i1-IQ2_M.gguf new file mode 100644 index 0000000..34ed04d --- /dev/null +++ b/Mistral-quiet-star-demo.i1-IQ2_M.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:50df0ac210d649d03421741d77d01cdd349a386515c1a4c5e22562593005280d +size 2500713760 diff --git a/Mistral-quiet-star-demo.i1-IQ2_S.gguf b/Mistral-quiet-star-demo.i1-IQ2_S.gguf new file mode 100644 index 0000000..ae4ce18 --- /dev/null +++ b/Mistral-quiet-star-demo.i1-IQ2_S.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:93b33ee0cfdfb7edfbf7c695aaab57a980103471b00a373e2b536c20be438f69 +size 2310921504 diff --git a/Mistral-quiet-star-demo.i1-IQ2_XS.gguf b/Mistral-quiet-star-demo.i1-IQ2_XS.gguf new file mode 100644 index 0000000..c329163 --- /dev/null +++ b/Mistral-quiet-star-demo.i1-IQ2_XS.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6bb6f7900c20920997ac1f42a07e9923abaacb529e832204c448825b4504cc3a +size 2198256928 diff --git a/Mistral-quiet-star-demo.i1-IQ2_XXS.gguf b/Mistral-quiet-star-demo.i1-IQ2_XXS.gguf new file mode 100644 index 0000000..c07f355 --- /dev/null +++ b/Mistral-quiet-star-demo.i1-IQ2_XXS.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2c2e3574e8380c9da585dfffb667fe9678a328736b7dd63b3f3a0c2a67717d3e +size 1991687456 diff --git a/Mistral-quiet-star-demo.i1-IQ3_M.gguf b/Mistral-quiet-star-demo.i1-IQ3_M.gguf new file mode 100644 index 0000000..09d6f64 --- /dev/null +++ b/Mistral-quiet-star-demo.i1-IQ3_M.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b0e4c90a24fcc823551dfd2cf637cb091a9ff514b01a5c6249e1a21ffe81b82d +size 3284892960 diff --git a/Mistral-quiet-star-demo.i1-IQ3_S.gguf b/Mistral-quiet-star-demo.i1-IQ3_S.gguf new file mode 100644 index 0000000..6c005b1 --- /dev/null +++ b/Mistral-quiet-star-demo.i1-IQ3_S.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f9c00106be17e150deca2d1466e531be2f560d39fab234889f700215d585d6e7 +size 3182394656 diff --git a/Mistral-quiet-star-demo.i1-IQ3_XS.gguf b/Mistral-quiet-star-demo.i1-IQ3_XS.gguf new file mode 100644 index 0000000..a81de88 --- /dev/null +++ b/Mistral-quiet-star-demo.i1-IQ3_XS.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:efc9693a2c0bc16b8c088399be51f36431fc45257a2abeea68ef4d0355934fc5 +size 3018816800 diff --git a/Mistral-quiet-star-demo.i1-IQ3_XXS.gguf b/Mistral-quiet-star-demo.i1-IQ3_XXS.gguf new file mode 100644 index 0000000..6ea94b6 --- /dev/null +++ b/Mistral-quiet-star-demo.i1-IQ3_XXS.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7387e26d0ef3defae405892cf6b239b7b00f6ba09be673affac4a7cd5324c6e4 +size 2827345184 diff --git a/Mistral-quiet-star-demo.i1-IQ4_XS.gguf b/Mistral-quiet-star-demo.i1-IQ4_XS.gguf new file mode 100644 index 0000000..45eccdc --- /dev/null +++ b/Mistral-quiet-star-demo.i1-IQ4_XS.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1111ba60fe138d8b4633568a750a9918be1e4d3442148dbe982e649145f103da +size 3907689760 diff --git a/Mistral-quiet-star-demo.i1-Q2_K.gguf b/Mistral-quiet-star-demo.i1-Q2_K.gguf new file mode 100644 index 0000000..eec0b7a --- /dev/null +++ b/Mistral-quiet-star-demo.i1-Q2_K.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d72b4ac30e452a923551bebc6203892622c5b04e82575cad4c361b9b0a4a79f1 +size 2719243552 diff --git a/Mistral-quiet-star-demo.i1-Q3_K_L.gguf b/Mistral-quiet-star-demo.i1-Q3_K_L.gguf new file mode 100644 index 0000000..5140af9 --- /dev/null +++ b/Mistral-quiet-star-demo.i1-Q3_K_L.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5798d6e05cc99372b9dc96d08fe9205812f74bc86ddcef9ad5b4410aa4e96273 +size 3822026016 diff --git a/Mistral-quiet-star-demo.i1-Q3_K_M.gguf b/Mistral-quiet-star-demo.i1-Q3_K_M.gguf new file mode 100644 index 0000000..1c7947e --- /dev/null +++ b/Mistral-quiet-star-demo.i1-Q3_K_M.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7a87bc12c0e23d740e570d47c709d670d220db56df424f19ad5cca7c6c2d9666 +size 3518987552 diff --git a/Mistral-quiet-star-demo.i1-Q3_K_S.gguf b/Mistral-quiet-star-demo.i1-Q3_K_S.gguf new file mode 100644 index 0000000..26e82d5 --- /dev/null +++ b/Mistral-quiet-star-demo.i1-Q3_K_S.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8964f226019fbbce5a7c39b525a74f16a4a426d2a839deea6e39a8f65b979742 +size 3164568864 diff --git a/Mistral-quiet-star-demo.i1-Q4_0.gguf b/Mistral-quiet-star-demo.i1-Q4_0.gguf new file mode 100644 index 0000000..03551c9 --- /dev/null +++ b/Mistral-quiet-star-demo.i1-Q4_0.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:88603e146a604d9203adf76bc8a582a45a7b9e0730afe4059278c106f7302706 +size 4123598112 diff --git a/Mistral-quiet-star-demo.i1-Q4_0_4_4.gguf b/Mistral-quiet-star-demo.i1-Q4_0_4_4.gguf new file mode 100644 index 0000000..c8af391 --- /dev/null +++ b/Mistral-quiet-star-demo.i1-Q4_0_4_4.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ae3a4d374eb8a568db9c1711bc43a7deafe8b392f5c91f20de31d628ecf6c2d +size 4108918048 diff --git a/Mistral-quiet-star-demo.i1-Q4_0_4_8.gguf b/Mistral-quiet-star-demo.i1-Q4_0_4_8.gguf new file mode 100644 index 0000000..b0952e6 --- /dev/null +++ b/Mistral-quiet-star-demo.i1-Q4_0_4_8.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a01d3b96a02bb179fef1c94f2204a5051f7d229acf82cdcba0d3700b1a231889 +size 4108918048 diff --git a/Mistral-quiet-star-demo.i1-Q4_0_8_8.gguf b/Mistral-quiet-star-demo.i1-Q4_0_8_8.gguf new file mode 100644 index 0000000..ea5d2dc --- /dev/null +++ b/Mistral-quiet-star-demo.i1-Q4_0_8_8.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:100ddeab85a35c277bf2e99743b4675bae6439ffb9837b22cc7d9666711922bf +size 4108918048 diff --git a/Mistral-quiet-star-demo.i1-Q4_K_M.gguf b/Mistral-quiet-star-demo.i1-Q4_K_M.gguf new file mode 100644 index 0000000..dbe526c --- /dev/null +++ b/Mistral-quiet-star-demo.i1-Q4_K_M.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e86d9b23fbe7a147281422bbb42ac8661ec31530cb378eedbc97eaf008438476 +size 4368440608 diff --git a/Mistral-quiet-star-demo.i1-Q4_K_S.gguf b/Mistral-quiet-star-demo.i1-Q4_K_S.gguf new file mode 100644 index 0000000..89e36a4 --- /dev/null +++ b/Mistral-quiet-star-demo.i1-Q4_K_S.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5d5607f03ba90c56a4330fae89b85187a544bc14d45698fb263e0e54df6ca03d +size 4140375328 diff --git a/Mistral-quiet-star-demo.i1-Q5_K_M.gguf b/Mistral-quiet-star-demo.i1-Q5_K_M.gguf new file mode 100644 index 0000000..05555ac --- /dev/null +++ b/Mistral-quiet-star-demo.i1-Q5_K_M.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dce7a6a692c3ff954d8c76a23a043de9ef7fa8a67f20e4351506c61dd5c40635 +size 5131410720 diff --git a/Mistral-quiet-star-demo.i1-Q5_K_S.gguf b/Mistral-quiet-star-demo.i1-Q5_K_S.gguf new file mode 100644 index 0000000..c4fdb58 --- /dev/null +++ b/Mistral-quiet-star-demo.i1-Q5_K_S.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:78d18a76ff082922fa86047696ad381e5e23ff275dc928fa9d04c4c3523507a7 +size 4997717280 diff --git a/Mistral-quiet-star-demo.i1-Q6_K.gguf b/Mistral-quiet-star-demo.i1-Q6_K.gguf new file mode 100644 index 0000000..8025bc2 --- /dev/null +++ b/Mistral-quiet-star-demo.i1-Q6_K.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:36f96ef5dc9a9bcf57c6392a27657c53df63457f09c9999d04f348614a4ea6d0 +size 5942066464 diff --git a/README.md b/README.md new file mode 100644 index 0000000..656be66 --- /dev/null +++ b/README.md @@ -0,0 +1,84 @@ +--- +base_model: liminerity/Mistral-quiet-star-demo +datasets: +- gate369/Alpaca-Star +language: +- en +library_name: transformers +license: apache-2.0 +quantized_by: mradermacher +tags: +- text-generation-inference +- transformers +- unsloth +- mistral +- trl +--- +## About + + + + + + +weighted/imatrix quants of https://huggingface.co/liminerity/Mistral-quiet-star-demo + + +static quants are available at https://huggingface.co/mradermacher/Mistral-quiet-star-demo-GGUF +## Usage + +If you are unsure how to use GGUF files, refer to one of [TheBloke's +READMEs](https://huggingface.co/TheBloke/KafkaLM-70B-German-V0.1-GGUF) for +more details, including on how to concatenate multi-part files. + +## Provided Quants + +(sorted by size, not necessarily quality. IQ-quants are often preferable over similar sized non-IQ quants) + +| Link | Type | Size/GB | Notes | +|:-----|:-----|--------:|:------| +| [GGUF](https://huggingface.co/mradermacher/Mistral-quiet-star-demo-i1-GGUF/resolve/main/Mistral-quiet-star-demo.i1-IQ1_S.gguf) | i1-IQ1_S | 1.7 | for the desperate | +| [GGUF](https://huggingface.co/mradermacher/Mistral-quiet-star-demo-i1-GGUF/resolve/main/Mistral-quiet-star-demo.i1-IQ1_M.gguf) | i1-IQ1_M | 1.9 | mostly desperate | +| [GGUF](https://huggingface.co/mradermacher/Mistral-quiet-star-demo-i1-GGUF/resolve/main/Mistral-quiet-star-demo.i1-IQ2_XXS.gguf) | i1-IQ2_XXS | 2.1 | | +| [GGUF](https://huggingface.co/mradermacher/Mistral-quiet-star-demo-i1-GGUF/resolve/main/Mistral-quiet-star-demo.i1-IQ2_XS.gguf) | i1-IQ2_XS | 2.3 | | +| [GGUF](https://huggingface.co/mradermacher/Mistral-quiet-star-demo-i1-GGUF/resolve/main/Mistral-quiet-star-demo.i1-IQ2_S.gguf) | i1-IQ2_S | 2.4 | | +| [GGUF](https://huggingface.co/mradermacher/Mistral-quiet-star-demo-i1-GGUF/resolve/main/Mistral-quiet-star-demo.i1-IQ2_M.gguf) | i1-IQ2_M | 2.6 | | +| [GGUF](https://huggingface.co/mradermacher/Mistral-quiet-star-demo-i1-GGUF/resolve/main/Mistral-quiet-star-demo.i1-Q2_K.gguf) | i1-Q2_K | 2.8 | IQ3_XXS probably better | +| [GGUF](https://huggingface.co/mradermacher/Mistral-quiet-star-demo-i1-GGUF/resolve/main/Mistral-quiet-star-demo.i1-IQ3_XXS.gguf) | i1-IQ3_XXS | 2.9 | lower quality | +| [GGUF](https://huggingface.co/mradermacher/Mistral-quiet-star-demo-i1-GGUF/resolve/main/Mistral-quiet-star-demo.i1-IQ3_XS.gguf) | i1-IQ3_XS | 3.1 | | +| [GGUF](https://huggingface.co/mradermacher/Mistral-quiet-star-demo-i1-GGUF/resolve/main/Mistral-quiet-star-demo.i1-Q3_K_S.gguf) | i1-Q3_K_S | 3.3 | IQ3_XS probably better | +| [GGUF](https://huggingface.co/mradermacher/Mistral-quiet-star-demo-i1-GGUF/resolve/main/Mistral-quiet-star-demo.i1-IQ3_S.gguf) | i1-IQ3_S | 3.3 | beats Q3_K* | +| [GGUF](https://huggingface.co/mradermacher/Mistral-quiet-star-demo-i1-GGUF/resolve/main/Mistral-quiet-star-demo.i1-IQ3_M.gguf) | i1-IQ3_M | 3.4 | | +| [GGUF](https://huggingface.co/mradermacher/Mistral-quiet-star-demo-i1-GGUF/resolve/main/Mistral-quiet-star-demo.i1-Q3_K_M.gguf) | i1-Q3_K_M | 3.6 | IQ3_S probably better | +| [GGUF](https://huggingface.co/mradermacher/Mistral-quiet-star-demo-i1-GGUF/resolve/main/Mistral-quiet-star-demo.i1-Q3_K_L.gguf) | i1-Q3_K_L | 3.9 | IQ3_M probably better | +| [GGUF](https://huggingface.co/mradermacher/Mistral-quiet-star-demo-i1-GGUF/resolve/main/Mistral-quiet-star-demo.i1-IQ4_XS.gguf) | i1-IQ4_XS | 4.0 | | +| [GGUF](https://huggingface.co/mradermacher/Mistral-quiet-star-demo-i1-GGUF/resolve/main/Mistral-quiet-star-demo.i1-Q4_0_4_4.gguf) | i1-Q4_0_4_4 | 4.2 | fast on arm, low quality | +| [GGUF](https://huggingface.co/mradermacher/Mistral-quiet-star-demo-i1-GGUF/resolve/main/Mistral-quiet-star-demo.i1-Q4_0_4_8.gguf) | i1-Q4_0_4_8 | 4.2 | fast on arm+i8mm, low quality | +| [GGUF](https://huggingface.co/mradermacher/Mistral-quiet-star-demo-i1-GGUF/resolve/main/Mistral-quiet-star-demo.i1-Q4_0_8_8.gguf) | i1-Q4_0_8_8 | 4.2 | fast on arm+sve, low quality | +| [GGUF](https://huggingface.co/mradermacher/Mistral-quiet-star-demo-i1-GGUF/resolve/main/Mistral-quiet-star-demo.i1-Q4_0.gguf) | i1-Q4_0 | 4.2 | fast, low quality | +| [GGUF](https://huggingface.co/mradermacher/Mistral-quiet-star-demo-i1-GGUF/resolve/main/Mistral-quiet-star-demo.i1-Q4_K_S.gguf) | i1-Q4_K_S | 4.2 | optimal size/speed/quality | +| [GGUF](https://huggingface.co/mradermacher/Mistral-quiet-star-demo-i1-GGUF/resolve/main/Mistral-quiet-star-demo.i1-Q4_K_M.gguf) | i1-Q4_K_M | 4.5 | fast, recommended | +| [GGUF](https://huggingface.co/mradermacher/Mistral-quiet-star-demo-i1-GGUF/resolve/main/Mistral-quiet-star-demo.i1-Q5_K_S.gguf) | i1-Q5_K_S | 5.1 | | +| [GGUF](https://huggingface.co/mradermacher/Mistral-quiet-star-demo-i1-GGUF/resolve/main/Mistral-quiet-star-demo.i1-Q5_K_M.gguf) | i1-Q5_K_M | 5.2 | | +| [GGUF](https://huggingface.co/mradermacher/Mistral-quiet-star-demo-i1-GGUF/resolve/main/Mistral-quiet-star-demo.i1-Q6_K.gguf) | i1-Q6_K | 6.0 | practically like static Q6_K | + +Here is a handy graph by ikawrakow comparing some lower-quality quant +types (lower is better): + +![image.png](https://www.nethype.de/huggingface_embed/quantpplgraph.png) + +And here are Artefact2's thoughts on the matter: +https://gist.github.com/Artefact2/b5f810600771265fc1e39442288e8ec9 + +## FAQ / Model Request + +See https://huggingface.co/mradermacher/model_requests for some answers to +questions you might have and/or if you want some other model quantized. + +## Thanks + +I thank my company, [nethype GmbH](https://www.nethype.de/), for letting +me use its servers and providing upgrades to my workstation to enable +this work in my free time. Additional thanks to [@nicoboss](https://huggingface.co/nicoboss) for giving me access to his private supercomputer, enabling me to provide many more imatrix quants, at much higher quality, than I would otherwise be able to. + + diff --git a/imatrix.dat b/imatrix.dat new file mode 100644 index 0000000..2c91882 --- /dev/null +++ b/imatrix.dat @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b9ea672bf84144386418dd65c82dd0c428bfc2500b1245e12a2dfd981bda324d +size 4988157