From e6b1c88c6eadb2ba71a26757c7b8c4b30b72af44 Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Sun, 9 Aug 2026 13:27:16 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: mradermacher/HuatuoGPT-3-8B-i1-GGUF Source: Original Platform --- .gitattributes | 60 +++++++++++++++++++++++ HuatuoGPT-3-8B.i1-IQ1_M.gguf | 3 ++ HuatuoGPT-3-8B.i1-IQ1_S.gguf | 3 ++ HuatuoGPT-3-8B.i1-IQ2_M.gguf | 3 ++ HuatuoGPT-3-8B.i1-IQ2_S.gguf | 3 ++ HuatuoGPT-3-8B.i1-IQ2_XS.gguf | 3 ++ HuatuoGPT-3-8B.i1-IQ2_XXS.gguf | 3 ++ HuatuoGPT-3-8B.i1-IQ3_M.gguf | 3 ++ HuatuoGPT-3-8B.i1-IQ3_S.gguf | 3 ++ HuatuoGPT-3-8B.i1-IQ3_XS.gguf | 3 ++ HuatuoGPT-3-8B.i1-IQ3_XXS.gguf | 3 ++ HuatuoGPT-3-8B.i1-IQ4_NL.gguf | 3 ++ HuatuoGPT-3-8B.i1-IQ4_XS.gguf | 3 ++ HuatuoGPT-3-8B.i1-Q2_K.gguf | 3 ++ HuatuoGPT-3-8B.i1-Q2_K_S.gguf | 3 ++ HuatuoGPT-3-8B.i1-Q3_K_L.gguf | 3 ++ HuatuoGPT-3-8B.i1-Q3_K_M.gguf | 3 ++ HuatuoGPT-3-8B.i1-Q3_K_S.gguf | 3 ++ HuatuoGPT-3-8B.i1-Q4_0.gguf | 3 ++ HuatuoGPT-3-8B.i1-Q4_1.gguf | 3 ++ HuatuoGPT-3-8B.i1-Q4_K_M.gguf | 3 ++ HuatuoGPT-3-8B.i1-Q4_K_S.gguf | 3 ++ HuatuoGPT-3-8B.i1-Q5_K_M.gguf | 3 ++ HuatuoGPT-3-8B.i1-Q5_K_S.gguf | 3 ++ HuatuoGPT-3-8B.i1-Q6_K.gguf | 3 ++ HuatuoGPT-3-8B.imatrix.gguf | 3 ++ README.md | 89 ++++++++++++++++++++++++++++++++++ 27 files changed, 224 insertions(+) create mode 100644 .gitattributes create mode 100644 HuatuoGPT-3-8B.i1-IQ1_M.gguf create mode 100644 HuatuoGPT-3-8B.i1-IQ1_S.gguf create mode 100644 HuatuoGPT-3-8B.i1-IQ2_M.gguf create mode 100644 HuatuoGPT-3-8B.i1-IQ2_S.gguf create mode 100644 HuatuoGPT-3-8B.i1-IQ2_XS.gguf create mode 100644 HuatuoGPT-3-8B.i1-IQ2_XXS.gguf create mode 100644 HuatuoGPT-3-8B.i1-IQ3_M.gguf create mode 100644 HuatuoGPT-3-8B.i1-IQ3_S.gguf create mode 100644 HuatuoGPT-3-8B.i1-IQ3_XS.gguf create mode 100644 HuatuoGPT-3-8B.i1-IQ3_XXS.gguf create mode 100644 HuatuoGPT-3-8B.i1-IQ4_NL.gguf create mode 100644 HuatuoGPT-3-8B.i1-IQ4_XS.gguf create mode 100644 HuatuoGPT-3-8B.i1-Q2_K.gguf create mode 100644 HuatuoGPT-3-8B.i1-Q2_K_S.gguf create mode 100644 HuatuoGPT-3-8B.i1-Q3_K_L.gguf create mode 100644 HuatuoGPT-3-8B.i1-Q3_K_M.gguf create mode 100644 HuatuoGPT-3-8B.i1-Q3_K_S.gguf create mode 100644 HuatuoGPT-3-8B.i1-Q4_0.gguf create mode 100644 HuatuoGPT-3-8B.i1-Q4_1.gguf create mode 100644 HuatuoGPT-3-8B.i1-Q4_K_M.gguf create mode 100644 HuatuoGPT-3-8B.i1-Q4_K_S.gguf create mode 100644 HuatuoGPT-3-8B.i1-Q5_K_M.gguf create mode 100644 HuatuoGPT-3-8B.i1-Q5_K_S.gguf create mode 100644 HuatuoGPT-3-8B.i1-Q6_K.gguf create mode 100644 HuatuoGPT-3-8B.imatrix.gguf create mode 100644 README.md diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..be37bba --- /dev/null +++ b/.gitattributes @@ -0,0 +1,60 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +HuatuoGPT-3-8B.imatrix.gguf filter=lfs diff=lfs merge=lfs -text +HuatuoGPT-3-8B.i1-Q2_K.gguf filter=lfs diff=lfs merge=lfs -text +HuatuoGPT-3-8B.i1-IQ3_M.gguf filter=lfs diff=lfs merge=lfs -text +HuatuoGPT-3-8B.i1-Q4_K_S.gguf filter=lfs diff=lfs merge=lfs -text +HuatuoGPT-3-8B.i1-IQ3_XXS.gguf filter=lfs diff=lfs merge=lfs -text +HuatuoGPT-3-8B.i1-IQ4_NL.gguf filter=lfs diff=lfs merge=lfs -text +HuatuoGPT-3-8B.i1-Q3_K_M.gguf filter=lfs diff=lfs merge=lfs -text +HuatuoGPT-3-8B.i1-Q6_K.gguf filter=lfs diff=lfs merge=lfs -text +HuatuoGPT-3-8B.i1-Q4_K_M.gguf filter=lfs diff=lfs merge=lfs -text +HuatuoGPT-3-8B.i1-IQ2_M.gguf filter=lfs diff=lfs merge=lfs -text +HuatuoGPT-3-8B.i1-Q2_K_S.gguf filter=lfs diff=lfs merge=lfs -text +HuatuoGPT-3-8B.i1-IQ4_XS.gguf filter=lfs diff=lfs merge=lfs -text +HuatuoGPT-3-8B.i1-Q3_K_L.gguf filter=lfs diff=lfs merge=lfs -text +HuatuoGPT-3-8B.i1-IQ1_M.gguf filter=lfs diff=lfs merge=lfs -text +HuatuoGPT-3-8B.i1-IQ2_XS.gguf filter=lfs diff=lfs merge=lfs -text +HuatuoGPT-3-8B.i1-Q3_K_S.gguf filter=lfs diff=lfs merge=lfs -text +HuatuoGPT-3-8B.i1-Q5_K_S.gguf filter=lfs diff=lfs merge=lfs -text +HuatuoGPT-3-8B.i1-IQ2_XXS.gguf filter=lfs diff=lfs merge=lfs -text +HuatuoGPT-3-8B.i1-IQ2_S.gguf filter=lfs diff=lfs merge=lfs -text +HuatuoGPT-3-8B.i1-IQ1_S.gguf filter=lfs diff=lfs merge=lfs -text +HuatuoGPT-3-8B.i1-Q5_K_M.gguf filter=lfs diff=lfs merge=lfs -text +HuatuoGPT-3-8B.i1-IQ3_XS.gguf filter=lfs diff=lfs merge=lfs -text +HuatuoGPT-3-8B.i1-Q4_0.gguf filter=lfs diff=lfs merge=lfs -text +HuatuoGPT-3-8B.i1-Q4_1.gguf filter=lfs diff=lfs merge=lfs -text +HuatuoGPT-3-8B.i1-IQ3_S.gguf filter=lfs diff=lfs merge=lfs -text diff --git a/HuatuoGPT-3-8B.i1-IQ1_M.gguf b/HuatuoGPT-3-8B.i1-IQ1_M.gguf new file mode 100644 index 0000000..ff9be7b --- /dev/null +++ b/HuatuoGPT-3-8B.i1-IQ1_M.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f15b0e20127ec2e0bb862c382c7175f16ef9753758bddf45b76ee364e3cbb141 +size 2256148448 diff --git a/HuatuoGPT-3-8B.i1-IQ1_S.gguf b/HuatuoGPT-3-8B.i1-IQ1_S.gguf new file mode 100644 index 0000000..673d8eb --- /dev/null +++ b/HuatuoGPT-3-8B.i1-IQ1_S.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c3355a4ee578a72f5f8f0d573ad09dc98b3f6ccdf219ee203c8fc32239e3a2ca +size 2115770336 diff --git a/HuatuoGPT-3-8B.i1-IQ2_M.gguf b/HuatuoGPT-3-8B.i1-IQ2_M.gguf new file mode 100644 index 0000000..eea7f4a --- /dev/null +++ b/HuatuoGPT-3-8B.i1-IQ2_M.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:36cb509682367eff992e4d8e6ec3b56d27d8561c512ae9b1aece70e5210847c0 +size 3051915232 diff --git a/HuatuoGPT-3-8B.i1-IQ2_S.gguf b/HuatuoGPT-3-8B.i1-IQ2_S.gguf new file mode 100644 index 0000000..a0b5b08 --- /dev/null +++ b/HuatuoGPT-3-8B.i1-IQ2_S.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:69599823d08120bcd80e8e4714ca3f3f5b017b34b94f41b3a3d4171ab07636f3 +size 2864744416 diff --git a/HuatuoGPT-3-8B.i1-IQ2_XS.gguf b/HuatuoGPT-3-8B.i1-IQ2_XS.gguf new file mode 100644 index 0000000..51cd492 --- /dev/null +++ b/HuatuoGPT-3-8B.i1-IQ2_XS.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:59f0cd938b3d895e5f0dc64878ba1dec148c4fac1fd73c636719d41e8a1000f0 +size 2696157152 diff --git a/HuatuoGPT-3-8B.i1-IQ2_XXS.gguf b/HuatuoGPT-3-8B.i1-IQ2_XXS.gguf new file mode 100644 index 0000000..9b2685d --- /dev/null +++ b/HuatuoGPT-3-8B.i1-IQ2_XXS.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b01a18a1e59377c583c01dbd9661485188cf0deb53eac787c6039de267a16c27 +size 2490111968 diff --git a/HuatuoGPT-3-8B.i1-IQ3_M.gguf b/HuatuoGPT-3-8B.i1-IQ3_M.gguf new file mode 100644 index 0000000..16a9316 --- /dev/null +++ b/HuatuoGPT-3-8B.i1-IQ3_M.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:819197d936c8b663165f61613b73cd6bc44a9e263a5e6957e0277e7580bb486e +size 3896621024 diff --git a/HuatuoGPT-3-8B.i1-IQ3_S.gguf b/HuatuoGPT-3-8B.i1-IQ3_S.gguf new file mode 100644 index 0000000..b1e2c92 --- /dev/null +++ b/HuatuoGPT-3-8B.i1-IQ3_S.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e81312d1bb1a64e01444335b2604aa53d9f343d981086ae907916ac1f150da91 +size 3789666272 diff --git a/HuatuoGPT-3-8B.i1-IQ3_XS.gguf b/HuatuoGPT-3-8B.i1-IQ3_XS.gguf new file mode 100644 index 0000000..d10ee38 --- /dev/null +++ b/HuatuoGPT-3-8B.i1-IQ3_XS.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4e81ef64709a3d73bfc9ca0adb0417bf180a147e1975d1d22d2817aa79db28e7 +size 3626874848 diff --git a/HuatuoGPT-3-8B.i1-IQ3_XXS.gguf b/HuatuoGPT-3-8B.i1-IQ3_XXS.gguf new file mode 100644 index 0000000..a5ae409 --- /dev/null +++ b/HuatuoGPT-3-8B.i1-IQ3_XXS.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dcbb3ba6568a2ba92605ede44bd417e54dbd5c0703451e4a24097df68bec7df5 +size 3369633760 diff --git a/HuatuoGPT-3-8B.i1-IQ4_NL.gguf b/HuatuoGPT-3-8B.i1-IQ4_NL.gguf new file mode 100644 index 0000000..4e9bfa6 --- /dev/null +++ b/HuatuoGPT-3-8B.i1-IQ4_NL.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c3ac68eabc3125c7f531d1643374ac71aba551374ada2353767d05f4b81a08ff +size 4793624544 diff --git a/HuatuoGPT-3-8B.i1-IQ4_XS.gguf b/HuatuoGPT-3-8B.i1-IQ4_XS.gguf new file mode 100644 index 0000000..962dce3 --- /dev/null +++ b/HuatuoGPT-3-8B.i1-IQ4_XS.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c46797a5c8f97b23e2c4933f6cbd6f75d5574d19b923715ff7d274fe8d75426f +size 4561840096 diff --git a/HuatuoGPT-3-8B.i1-Q2_K.gguf b/HuatuoGPT-3-8B.i1-Q2_K.gguf new file mode 100644 index 0000000..7b2ab34 --- /dev/null +++ b/HuatuoGPT-3-8B.i1-Q2_K.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d4bddf346502829c48a8bf06b7fbf6013cf2db3e4aca587355c45ade2869a42c +size 3281733600 diff --git a/HuatuoGPT-3-8B.i1-Q2_K_S.gguf b/HuatuoGPT-3-8B.i1-Q2_K_S.gguf new file mode 100644 index 0000000..aced9b6 --- /dev/null +++ b/HuatuoGPT-3-8B.i1-Q2_K_S.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2c999c0647bc7c6abf3cf8c6bee7101f66d8184ca2a8de575753833e94ef57e3 +size 3083552736 diff --git a/HuatuoGPT-3-8B.i1-Q3_K_L.gguf b/HuatuoGPT-3-8B.i1-Q3_K_L.gguf new file mode 100644 index 0000000..91fbc68 --- /dev/null +++ b/HuatuoGPT-3-8B.i1-Q3_K_L.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:865d58b90a5150205ee81eb1769dcae4a81aa6d2883f900e7dfd8d413d44bcd8 +size 4431394784 diff --git a/HuatuoGPT-3-8B.i1-Q3_K_M.gguf b/HuatuoGPT-3-8B.i1-Q3_K_M.gguf new file mode 100644 index 0000000..4df5e88 --- /dev/null +++ b/HuatuoGPT-3-8B.i1-Q3_K_M.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:08847944608ed38e67c32c40855bed2144bf0f8d010a05f274d2507d2834b899 +size 4124162016 diff --git a/HuatuoGPT-3-8B.i1-Q3_K_S.gguf b/HuatuoGPT-3-8B.i1-Q3_K_S.gguf new file mode 100644 index 0000000..f6b0fcf --- /dev/null +++ b/HuatuoGPT-3-8B.i1-Q3_K_S.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bbb93e35c57bbb7998e66a2d33943d25823d70b471b95bb03c6acacfdeb69f62 +size 3769612256 diff --git a/HuatuoGPT-3-8B.i1-Q4_0.gguf b/HuatuoGPT-3-8B.i1-Q4_0.gguf new file mode 100644 index 0000000..a4d4013 --- /dev/null +++ b/HuatuoGPT-3-8B.i1-Q4_0.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3170d5f680ad8ad9b0246ff8e44a1cfd72167388bd5e89a98200ff182b1d7bb9 +size 4787333088 diff --git a/HuatuoGPT-3-8B.i1-Q4_1.gguf b/HuatuoGPT-3-8B.i1-Q4_1.gguf new file mode 100644 index 0000000..abc24c2 --- /dev/null +++ b/HuatuoGPT-3-8B.i1-Q4_1.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f908828df434c4c02214101f58702102d6f953d8f8b8db147a8a82d38f03b10c +size 5247756256 diff --git a/HuatuoGPT-3-8B.i1-Q4_K_M.gguf b/HuatuoGPT-3-8B.i1-Q4_K_M.gguf new file mode 100644 index 0000000..7e2c51f --- /dev/null +++ b/HuatuoGPT-3-8B.i1-Q4_K_M.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f86b5ef0f8779eb61c3bf2a92b8e9b3330595c9424c07cf21ab8f0c00ed2ff4f +size 5027784672 diff --git a/HuatuoGPT-3-8B.i1-Q4_K_S.gguf b/HuatuoGPT-3-8B.i1-Q4_K_S.gguf new file mode 100644 index 0000000..4d5ad3c --- /dev/null +++ b/HuatuoGPT-3-8B.i1-Q4_K_S.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:603c2d5b373b6727f7d370ba358785763209400dc83ede4b06ceff068d9431f2 +size 4802013152 diff --git a/HuatuoGPT-3-8B.i1-Q5_K_M.gguf b/HuatuoGPT-3-8B.i1-Q5_K_M.gguf new file mode 100644 index 0000000..82f5096 --- /dev/null +++ b/HuatuoGPT-3-8B.i1-Q5_K_M.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:97922023fab74ca7c417842afde4a0cbfd51dd3415ec3ff47137396190afe43c +size 5851113440 diff --git a/HuatuoGPT-3-8B.i1-Q5_K_S.gguf b/HuatuoGPT-3-8B.i1-Q5_K_S.gguf new file mode 100644 index 0000000..51d44df --- /dev/null +++ b/HuatuoGPT-3-8B.i1-Q5_K_S.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:09f462b29793ce8fd8552b4520d165d55e062f06e1756405a6f20aac8d40c53c +size 5720762336 diff --git a/HuatuoGPT-3-8B.i1-Q6_K.gguf b/HuatuoGPT-3-8B.i1-Q6_K.gguf new file mode 100644 index 0000000..4fd87de --- /dev/null +++ b/HuatuoGPT-3-8B.i1-Q6_K.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d393d2240b352900665f527220cf566cebebb4feeb3607ec4bfef025ed55b16d +size 6725900256 diff --git a/HuatuoGPT-3-8B.imatrix.gguf b/HuatuoGPT-3-8B.imatrix.gguf new file mode 100644 index 0000000..70ad8fb --- /dev/null +++ b/HuatuoGPT-3-8B.imatrix.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7bde9f6542edc321b44079a364d19886dd2ac25bea70c72de873654e56c1e874 +size 5347200 diff --git a/README.md b/README.md new file mode 100644 index 0000000..de88b6f --- /dev/null +++ b/README.md @@ -0,0 +1,89 @@ +--- +base_model: FreedomIntelligence/HuatuoGPT-3-8B +language: +- en +- zh +library_name: transformers +license: apache-2.0 +mradermacher: + readme_rev: 1 +quantized_by: mradermacher +tags: +- medical +- reasoning +--- +## About + + + + + + + + + +weighted/imatrix quants of https://huggingface.co/FreedomIntelligence/HuatuoGPT-3-8B + + + +***For a convenient overview and download list, visit our [model page for this model](https://hf.tst.eu/model#HuatuoGPT-3-8B-i1-GGUF).*** + +static quants are available at https://huggingface.co/mradermacher/HuatuoGPT-3-8B-GGUF +## Usage + +If you are unsure how to use GGUF files, refer to one of [TheBloke's +READMEs](https://huggingface.co/TheBloke/KafkaLM-70B-German-V0.1-GGUF) for +more details, including on how to concatenate multi-part files. + +## Provided Quants + +(sorted by size, not necessarily quality. IQ-quants are often preferable over similar sized non-IQ quants) + +| Link | Type | Size/GB | Notes | +|:-----|:-----|--------:|:------| +| [GGUF](https://huggingface.co/mradermacher/HuatuoGPT-3-8B-i1-GGUF/resolve/main/HuatuoGPT-3-8B.imatrix.gguf) | imatrix | 0.1 | imatrix file (for creating your own quants) | +| [GGUF](https://huggingface.co/mradermacher/HuatuoGPT-3-8B-i1-GGUF/resolve/main/HuatuoGPT-3-8B.i1-IQ1_S.gguf) | i1-IQ1_S | 2.2 | for the desperate | +| [GGUF](https://huggingface.co/mradermacher/HuatuoGPT-3-8B-i1-GGUF/resolve/main/HuatuoGPT-3-8B.i1-IQ1_M.gguf) | i1-IQ1_M | 2.4 | mostly desperate | +| [GGUF](https://huggingface.co/mradermacher/HuatuoGPT-3-8B-i1-GGUF/resolve/main/HuatuoGPT-3-8B.i1-IQ2_XXS.gguf) | i1-IQ2_XXS | 2.6 | | +| [GGUF](https://huggingface.co/mradermacher/HuatuoGPT-3-8B-i1-GGUF/resolve/main/HuatuoGPT-3-8B.i1-IQ2_XS.gguf) | i1-IQ2_XS | 2.8 | | +| [GGUF](https://huggingface.co/mradermacher/HuatuoGPT-3-8B-i1-GGUF/resolve/main/HuatuoGPT-3-8B.i1-IQ2_S.gguf) | i1-IQ2_S | 3.0 | | +| [GGUF](https://huggingface.co/mradermacher/HuatuoGPT-3-8B-i1-GGUF/resolve/main/HuatuoGPT-3-8B.i1-IQ2_M.gguf) | i1-IQ2_M | 3.2 | | +| [GGUF](https://huggingface.co/mradermacher/HuatuoGPT-3-8B-i1-GGUF/resolve/main/HuatuoGPT-3-8B.i1-Q2_K_S.gguf) | i1-Q2_K_S | 3.2 | very low quality | +| [GGUF](https://huggingface.co/mradermacher/HuatuoGPT-3-8B-i1-GGUF/resolve/main/HuatuoGPT-3-8B.i1-Q2_K.gguf) | i1-Q2_K | 3.4 | IQ3_XXS probably better | +| [GGUF](https://huggingface.co/mradermacher/HuatuoGPT-3-8B-i1-GGUF/resolve/main/HuatuoGPT-3-8B.i1-IQ3_XXS.gguf) | i1-IQ3_XXS | 3.5 | lower quality | +| [GGUF](https://huggingface.co/mradermacher/HuatuoGPT-3-8B-i1-GGUF/resolve/main/HuatuoGPT-3-8B.i1-IQ3_XS.gguf) | i1-IQ3_XS | 3.7 | | +| [GGUF](https://huggingface.co/mradermacher/HuatuoGPT-3-8B-i1-GGUF/resolve/main/HuatuoGPT-3-8B.i1-Q3_K_S.gguf) | i1-Q3_K_S | 3.9 | IQ3_XS probably better | +| [GGUF](https://huggingface.co/mradermacher/HuatuoGPT-3-8B-i1-GGUF/resolve/main/HuatuoGPT-3-8B.i1-IQ3_S.gguf) | i1-IQ3_S | 3.9 | beats Q3_K* | +| [GGUF](https://huggingface.co/mradermacher/HuatuoGPT-3-8B-i1-GGUF/resolve/main/HuatuoGPT-3-8B.i1-IQ3_M.gguf) | i1-IQ3_M | 4.0 | | +| [GGUF](https://huggingface.co/mradermacher/HuatuoGPT-3-8B-i1-GGUF/resolve/main/HuatuoGPT-3-8B.i1-Q3_K_M.gguf) | i1-Q3_K_M | 4.2 | IQ3_S probably better | +| [GGUF](https://huggingface.co/mradermacher/HuatuoGPT-3-8B-i1-GGUF/resolve/main/HuatuoGPT-3-8B.i1-Q3_K_L.gguf) | i1-Q3_K_L | 4.5 | IQ3_M probably better | +| [GGUF](https://huggingface.co/mradermacher/HuatuoGPT-3-8B-i1-GGUF/resolve/main/HuatuoGPT-3-8B.i1-IQ4_XS.gguf) | i1-IQ4_XS | 4.7 | | +| [GGUF](https://huggingface.co/mradermacher/HuatuoGPT-3-8B-i1-GGUF/resolve/main/HuatuoGPT-3-8B.i1-Q4_0.gguf) | i1-Q4_0 | 4.9 | fast, low quality | +| [GGUF](https://huggingface.co/mradermacher/HuatuoGPT-3-8B-i1-GGUF/resolve/main/HuatuoGPT-3-8B.i1-IQ4_NL.gguf) | i1-IQ4_NL | 4.9 | prefer IQ4_XS | +| [GGUF](https://huggingface.co/mradermacher/HuatuoGPT-3-8B-i1-GGUF/resolve/main/HuatuoGPT-3-8B.i1-Q4_K_S.gguf) | i1-Q4_K_S | 4.9 | optimal size/speed/quality | +| [GGUF](https://huggingface.co/mradermacher/HuatuoGPT-3-8B-i1-GGUF/resolve/main/HuatuoGPT-3-8B.i1-Q4_K_M.gguf) | i1-Q4_K_M | 5.1 | fast, recommended | +| [GGUF](https://huggingface.co/mradermacher/HuatuoGPT-3-8B-i1-GGUF/resolve/main/HuatuoGPT-3-8B.i1-Q4_1.gguf) | i1-Q4_1 | 5.3 | | +| [GGUF](https://huggingface.co/mradermacher/HuatuoGPT-3-8B-i1-GGUF/resolve/main/HuatuoGPT-3-8B.i1-Q5_K_S.gguf) | i1-Q5_K_S | 5.8 | | +| [GGUF](https://huggingface.co/mradermacher/HuatuoGPT-3-8B-i1-GGUF/resolve/main/HuatuoGPT-3-8B.i1-Q5_K_M.gguf) | i1-Q5_K_M | 6.0 | | +| [GGUF](https://huggingface.co/mradermacher/HuatuoGPT-3-8B-i1-GGUF/resolve/main/HuatuoGPT-3-8B.i1-Q6_K.gguf) | i1-Q6_K | 6.8 | practically like static Q6_K | + +Here is a handy graph by ikawrakow comparing some lower-quality quant +types (lower is better): + +![image.png](https://www.nethype.de/huggingface_embed/quantpplgraph.png) + +And here are Artefact2's thoughts on the matter: +https://gist.github.com/Artefact2/b5f810600771265fc1e39442288e8ec9 + +## FAQ / Model Request + +See https://huggingface.co/mradermacher/model_requests for some answers to +questions you might have and/or if you want some other model quantized. + +## Thanks + +I thank my company, [nethype GmbH](https://www.nethype.de/), for letting +me use its servers and providing upgrades to my workstation to enable +this work in my free time. Additional thanks to [@nicoboss](https://huggingface.co/nicoboss) for giving me access to his private supercomputer, enabling me to provide many more imatrix quants, at much higher quality, than I would otherwise be able to. + +