From 704c26e8d50aba97fc1660bb6625389dc0ae35d7 Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Mon, 12 Oct 2026 01:07:16 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: mradermacher/LLaMA-Pro-8B-Instruct-i1-GGUF Source: Original Platform --- .gitattributes | 60 ++++++++++++++++++++ LLaMA-Pro-8B-Instruct.i1-IQ1_M.gguf | 3 + LLaMA-Pro-8B-Instruct.i1-IQ1_S.gguf | 3 + LLaMA-Pro-8B-Instruct.i1-IQ2_M.gguf | 3 + LLaMA-Pro-8B-Instruct.i1-IQ2_S.gguf | 3 + LLaMA-Pro-8B-Instruct.i1-IQ2_XS.gguf | 3 + LLaMA-Pro-8B-Instruct.i1-IQ2_XXS.gguf | 3 + LLaMA-Pro-8B-Instruct.i1-IQ3_M.gguf | 3 + LLaMA-Pro-8B-Instruct.i1-IQ3_S.gguf | 3 + LLaMA-Pro-8B-Instruct.i1-IQ3_XS.gguf | 3 + LLaMA-Pro-8B-Instruct.i1-IQ3_XXS.gguf | 3 + LLaMA-Pro-8B-Instruct.i1-IQ4_XS.gguf | 3 + LLaMA-Pro-8B-Instruct.i1-Q2_K.gguf | 3 + LLaMA-Pro-8B-Instruct.i1-Q3_K_L.gguf | 3 + LLaMA-Pro-8B-Instruct.i1-Q3_K_M.gguf | 3 + LLaMA-Pro-8B-Instruct.i1-Q3_K_S.gguf | 3 + LLaMA-Pro-8B-Instruct.i1-Q4_0.gguf | 3 + LLaMA-Pro-8B-Instruct.i1-Q4_0_4_4.gguf | 3 + LLaMA-Pro-8B-Instruct.i1-Q4_0_4_8.gguf | 3 + LLaMA-Pro-8B-Instruct.i1-Q4_0_8_8.gguf | 3 + LLaMA-Pro-8B-Instruct.i1-Q4_K_M.gguf | 3 + LLaMA-Pro-8B-Instruct.i1-Q4_K_S.gguf | 3 + LLaMA-Pro-8B-Instruct.i1-Q5_K_M.gguf | 3 + LLaMA-Pro-8B-Instruct.i1-Q5_K_S.gguf | 3 + LLaMA-Pro-8B-Instruct.i1-Q6_K.gguf | 3 + README.md | 76 ++++++++++++++++++++++++++ imatrix.dat | 3 + 27 files changed, 211 insertions(+) create mode 100644 .gitattributes create mode 100644 LLaMA-Pro-8B-Instruct.i1-IQ1_M.gguf create mode 100644 LLaMA-Pro-8B-Instruct.i1-IQ1_S.gguf create mode 100644 LLaMA-Pro-8B-Instruct.i1-IQ2_M.gguf create mode 100644 LLaMA-Pro-8B-Instruct.i1-IQ2_S.gguf create mode 100644 LLaMA-Pro-8B-Instruct.i1-IQ2_XS.gguf create mode 100644 LLaMA-Pro-8B-Instruct.i1-IQ2_XXS.gguf create mode 100644 LLaMA-Pro-8B-Instruct.i1-IQ3_M.gguf create mode 100644 LLaMA-Pro-8B-Instruct.i1-IQ3_S.gguf create mode 100644 LLaMA-Pro-8B-Instruct.i1-IQ3_XS.gguf create mode 100644 LLaMA-Pro-8B-Instruct.i1-IQ3_XXS.gguf create mode 100644 LLaMA-Pro-8B-Instruct.i1-IQ4_XS.gguf create mode 100644 LLaMA-Pro-8B-Instruct.i1-Q2_K.gguf create mode 100644 LLaMA-Pro-8B-Instruct.i1-Q3_K_L.gguf create mode 100644 LLaMA-Pro-8B-Instruct.i1-Q3_K_M.gguf create mode 100644 LLaMA-Pro-8B-Instruct.i1-Q3_K_S.gguf create mode 100644 LLaMA-Pro-8B-Instruct.i1-Q4_0.gguf create mode 100644 LLaMA-Pro-8B-Instruct.i1-Q4_0_4_4.gguf create mode 100644 LLaMA-Pro-8B-Instruct.i1-Q4_0_4_8.gguf create mode 100644 LLaMA-Pro-8B-Instruct.i1-Q4_0_8_8.gguf create mode 100644 LLaMA-Pro-8B-Instruct.i1-Q4_K_M.gguf create mode 100644 LLaMA-Pro-8B-Instruct.i1-Q4_K_S.gguf create mode 100644 LLaMA-Pro-8B-Instruct.i1-Q5_K_M.gguf create mode 100644 LLaMA-Pro-8B-Instruct.i1-Q5_K_S.gguf create mode 100644 LLaMA-Pro-8B-Instruct.i1-Q6_K.gguf create mode 100644 README.md create mode 100644 imatrix.dat diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..9adac35 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,60 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +imatrix.dat filter=lfs diff=lfs merge=lfs -text +LLaMA-Pro-8B-Instruct.i1-Q2_K.gguf filter=lfs diff=lfs merge=lfs -text +LLaMA-Pro-8B-Instruct.i1-IQ3_M.gguf filter=lfs diff=lfs merge=lfs -text +LLaMA-Pro-8B-Instruct.i1-Q4_K_S.gguf filter=lfs diff=lfs merge=lfs -text +LLaMA-Pro-8B-Instruct.i1-IQ3_XXS.gguf filter=lfs diff=lfs merge=lfs -text +LLaMA-Pro-8B-Instruct.i1-Q3_K_M.gguf filter=lfs diff=lfs merge=lfs -text +LLaMA-Pro-8B-Instruct.i1-Q4_K_M.gguf filter=lfs diff=lfs merge=lfs -text +LLaMA-Pro-8B-Instruct.i1-IQ2_M.gguf filter=lfs diff=lfs merge=lfs -text +LLaMA-Pro-8B-Instruct.i1-Q6_K.gguf filter=lfs diff=lfs merge=lfs -text +LLaMA-Pro-8B-Instruct.i1-IQ1_M.gguf filter=lfs diff=lfs merge=lfs -text +LLaMA-Pro-8B-Instruct.i1-IQ4_XS.gguf filter=lfs diff=lfs merge=lfs -text +LLaMA-Pro-8B-Instruct.i1-Q3_K_S.gguf filter=lfs diff=lfs merge=lfs -text +LLaMA-Pro-8B-Instruct.i1-IQ2_XXS.gguf filter=lfs diff=lfs merge=lfs -text +LLaMA-Pro-8B-Instruct.i1-Q3_K_L.gguf filter=lfs diff=lfs merge=lfs -text +LLaMA-Pro-8B-Instruct.i1-IQ2_XS.gguf filter=lfs diff=lfs merge=lfs -text +LLaMA-Pro-8B-Instruct.i1-Q5_K_S.gguf filter=lfs diff=lfs merge=lfs -text +LLaMA-Pro-8B-Instruct.i1-IQ2_S.gguf filter=lfs diff=lfs merge=lfs -text +LLaMA-Pro-8B-Instruct.i1-Q4_0_4_4.gguf filter=lfs diff=lfs merge=lfs -text +LLaMA-Pro-8B-Instruct.i1-IQ1_S.gguf filter=lfs diff=lfs merge=lfs -text +LLaMA-Pro-8B-Instruct.i1-Q5_K_M.gguf filter=lfs diff=lfs merge=lfs -text +LLaMA-Pro-8B-Instruct.i1-Q4_0.gguf filter=lfs diff=lfs merge=lfs -text +LLaMA-Pro-8B-Instruct.i1-IQ3_XS.gguf filter=lfs diff=lfs merge=lfs -text +LLaMA-Pro-8B-Instruct.i1-Q4_0_4_8.gguf filter=lfs diff=lfs merge=lfs -text +LLaMA-Pro-8B-Instruct.i1-Q4_0_8_8.gguf filter=lfs diff=lfs merge=lfs -text +LLaMA-Pro-8B-Instruct.i1-IQ3_S.gguf filter=lfs diff=lfs merge=lfs -text diff --git a/LLaMA-Pro-8B-Instruct.i1-IQ1_M.gguf b/LLaMA-Pro-8B-Instruct.i1-IQ1_M.gguf new file mode 100644 index 0000000..ebc0889 --- /dev/null +++ b/LLaMA-Pro-8B-Instruct.i1-IQ1_M.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:224ff2eb2f23408a30d04ba222e19b8fade3e94d6471fe1548e1866e0ee351f6 +size 2030249280 diff --git a/LLaMA-Pro-8B-Instruct.i1-IQ1_S.gguf b/LLaMA-Pro-8B-Instruct.i1-IQ1_S.gguf new file mode 100644 index 0000000..852316a --- /dev/null +++ b/LLaMA-Pro-8B-Instruct.i1-IQ1_S.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0e0ba97145884c33be6dfcc21e7bd12a10ff229a9b44d21533223e8454aca941 +size 1877263680 diff --git a/LLaMA-Pro-8B-Instruct.i1-IQ2_M.gguf b/LLaMA-Pro-8B-Instruct.i1-IQ2_M.gguf new file mode 100644 index 0000000..4ba4976 --- /dev/null +++ b/LLaMA-Pro-8B-Instruct.i1-IQ2_M.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b47c071eaa57b410ac8fafd76444e6c18aff3afe478195691437bced338c5525 +size 2912896320 diff --git a/LLaMA-Pro-8B-Instruct.i1-IQ2_S.gguf b/LLaMA-Pro-8B-Instruct.i1-IQ2_S.gguf new file mode 100644 index 0000000..971759e --- /dev/null +++ b/LLaMA-Pro-8B-Instruct.i1-IQ2_S.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9ad06867067c4777af99a6d5d93df8aa353fb7ab6b736bd34e0204f32561f955 +size 2708915520 diff --git a/LLaMA-Pro-8B-Instruct.i1-IQ2_XS.gguf b/LLaMA-Pro-8B-Instruct.i1-IQ2_XS.gguf new file mode 100644 index 0000000..aea15f8 --- /dev/null +++ b/LLaMA-Pro-8B-Instruct.i1-IQ2_XS.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c7acf823fc2f911de890dfd512a880ffece35d85929ee8c86152108b5ba0317f +size 2510177600 diff --git a/LLaMA-Pro-8B-Instruct.i1-IQ2_XXS.gguf b/LLaMA-Pro-8B-Instruct.i1-IQ2_XXS.gguf new file mode 100644 index 0000000..e9a730b --- /dev/null +++ b/LLaMA-Pro-8B-Instruct.i1-IQ2_XXS.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fde30d92d186f1df797a18e4f6e3a6c4e880e70d333efbe49bc419c6ff428e38 +size 2285225280 diff --git a/LLaMA-Pro-8B-Instruct.i1-IQ3_M.gguf b/LLaMA-Pro-8B-Instruct.i1-IQ3_M.gguf new file mode 100644 index 0000000..62632e4 --- /dev/null +++ b/LLaMA-Pro-8B-Instruct.i1-IQ3_M.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5cba80ae66094b2935e641af788191fd574deae7c1b4e7dbb4dd5d7144ec0f3a +size 3852436800 diff --git a/LLaMA-Pro-8B-Instruct.i1-IQ3_S.gguf b/LLaMA-Pro-8B-Instruct.i1-IQ3_S.gguf new file mode 100644 index 0000000..247dac6 --- /dev/null +++ b/LLaMA-Pro-8B-Instruct.i1-IQ3_S.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6c3b3408d9c10e6355d3dc6f220d1fde13759b05bc6c5089d782edcb5272bb37 +size 3644237120 diff --git a/LLaMA-Pro-8B-Instruct.i1-IQ3_XS.gguf b/LLaMA-Pro-8B-Instruct.i1-IQ3_XS.gguf new file mode 100644 index 0000000..9901f8a --- /dev/null +++ b/LLaMA-Pro-8B-Instruct.i1-IQ3_XS.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8c48aaa21197d675d1046a1ee198db9d055b292cab621001bb44c7d62f96105c +size 3454510400 diff --git a/LLaMA-Pro-8B-Instruct.i1-IQ3_XXS.gguf b/LLaMA-Pro-8B-Instruct.i1-IQ3_XXS.gguf new file mode 100644 index 0000000..248dd71 --- /dev/null +++ b/LLaMA-Pro-8B-Instruct.i1-IQ3_XXS.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:013bd4cf8ed96176042e139a4e47feb3058db33e9534a279aa1bbc710287af5c +size 3194946880 diff --git a/LLaMA-Pro-8B-Instruct.i1-IQ4_XS.gguf b/LLaMA-Pro-8B-Instruct.i1-IQ4_XS.gguf new file mode 100644 index 0000000..7e05d86 --- /dev/null +++ b/LLaMA-Pro-8B-Instruct.i1-IQ4_XS.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0464273bfac852d8e493edf28b3607a892f959718a6d108c6f849cab33cb00f +size 4479698240 diff --git a/LLaMA-Pro-8B-Instruct.i1-Q2_K.gguf b/LLaMA-Pro-8B-Instruct.i1-Q2_K.gguf new file mode 100644 index 0000000..df92abe --- /dev/null +++ b/LLaMA-Pro-8B-Instruct.i1-Q2_K.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1b3f3b2f202ef61929ea52c8edce35b7cba1bbd3319023f1757204e731089c5c +size 3128264000 diff --git a/LLaMA-Pro-8B-Instruct.i1-Q3_K_L.gguf b/LLaMA-Pro-8B-Instruct.i1-Q3_K_L.gguf new file mode 100644 index 0000000..01e2b62 --- /dev/null +++ b/LLaMA-Pro-8B-Instruct.i1-Q3_K_L.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7e2288637d398eb1eb93b1d2ccd62836e8fb17717fbfaecd6b99a8c51611fcef +size 4455245120 diff --git a/LLaMA-Pro-8B-Instruct.i1-Q3_K_M.gguf b/LLaMA-Pro-8B-Instruct.i1-Q3_K_M.gguf new file mode 100644 index 0000000..995072b --- /dev/null +++ b/LLaMA-Pro-8B-Instruct.i1-Q3_K_M.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b8ed316944432b852ea3146509b48d0fa221e63789416a251e8c47db573c077a +size 4077495616 diff --git a/LLaMA-Pro-8B-Instruct.i1-Q3_K_S.gguf b/LLaMA-Pro-8B-Instruct.i1-Q3_K_S.gguf new file mode 100644 index 0000000..f3c0cc7 --- /dev/null +++ b/LLaMA-Pro-8B-Instruct.i1-Q3_K_S.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c334770058454b6e1a0cde7129c39569a07e32d9eccab31bade489e08f8e6024 +size 3644237120 diff --git a/LLaMA-Pro-8B-Instruct.i1-Q4_0.gguf b/LLaMA-Pro-8B-Instruct.i1-Q4_0.gguf new file mode 100644 index 0000000..b6929fd --- /dev/null +++ b/LLaMA-Pro-8B-Instruct.i1-Q4_0.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:081568b03d9ccff9371ae45a0f74e0ea3f491057b71cc4eab9ddb535531e3166 +size 4750853440 diff --git a/LLaMA-Pro-8B-Instruct.i1-Q4_0_4_4.gguf b/LLaMA-Pro-8B-Instruct.i1-Q4_0_4_4.gguf new file mode 100644 index 0000000..2bbe0f2 --- /dev/null +++ b/LLaMA-Pro-8B-Instruct.i1-Q4_0_4_4.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7cfc1fe547a4dfc88d27d92670a5a485786043676fc41303ab6586668d2f564e +size 4736763200 diff --git a/LLaMA-Pro-8B-Instruct.i1-Q4_0_4_8.gguf b/LLaMA-Pro-8B-Instruct.i1-Q4_0_4_8.gguf new file mode 100644 index 0000000..5169dc9 --- /dev/null +++ b/LLaMA-Pro-8B-Instruct.i1-Q4_0_4_8.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:831263dd7e3f9bf367c3fc14c7ebe9d29e4f5a9aa66e446e26e5798d324c4385 +size 4736763200 diff --git a/LLaMA-Pro-8B-Instruct.i1-Q4_0_8_8.gguf b/LLaMA-Pro-8B-Instruct.i1-Q4_0_8_8.gguf new file mode 100644 index 0000000..d51d3fa --- /dev/null +++ b/LLaMA-Pro-8B-Instruct.i1-Q4_0_8_8.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c7c4e18387438dc3d543674fdca591364616daeb49270a5cdf46008e09dc646e +size 4736763200 diff --git a/LLaMA-Pro-8B-Instruct.i1-Q4_K_M.gguf b/LLaMA-Pro-8B-Instruct.i1-Q4_K_M.gguf new file mode 100644 index 0000000..2b13500 --- /dev/null +++ b/LLaMA-Pro-8B-Instruct.i1-Q4_K_M.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f4e0a81e689a2e6ac920cb5da522c23c161bf5c80fd80b1c476da77f3cb93dcf +size 5055759680 diff --git a/LLaMA-Pro-8B-Instruct.i1-Q4_K_S.gguf b/LLaMA-Pro-8B-Instruct.i1-Q4_K_S.gguf new file mode 100644 index 0000000..a6cdb67 --- /dev/null +++ b/LLaMA-Pro-8B-Instruct.i1-Q4_K_S.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c19c1def16ffc8619d7c3bca97ca34746577a4972db1aa1b5df8f00c2e83fd71 +size 4773332288 diff --git a/LLaMA-Pro-8B-Instruct.i1-Q5_K_M.gguf b/LLaMA-Pro-8B-Instruct.i1-Q5_K_M.gguf new file mode 100644 index 0000000..3a9ea69 --- /dev/null +++ b/LLaMA-Pro-8B-Instruct.i1-Q5_K_M.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8e4ef7edf8b03879ed217c794f732a3ddcba8fe31f2ae2cbbe38e3fd64908c2e +size 5929354560 diff --git a/LLaMA-Pro-8B-Instruct.i1-Q5_K_S.gguf b/LLaMA-Pro-8B-Instruct.i1-Q5_K_S.gguf new file mode 100644 index 0000000..960a5df --- /dev/null +++ b/LLaMA-Pro-8B-Instruct.i1-Q5_K_S.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ce8ae160c46000fb8fd57d2e83fb431cdcd24b0d0167f4bfbd4f69679dc165e8 +size 5765023040 diff --git a/LLaMA-Pro-8B-Instruct.i1-Q6_K.gguf b/LLaMA-Pro-8B-Instruct.i1-Q6_K.gguf new file mode 100644 index 0000000..d7d807a --- /dev/null +++ b/LLaMA-Pro-8B-Instruct.i1-Q6_K.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f64c38aec470e8522d0e7af2a313bec1444335d53c35cb2cc8ec9bc2b6c83447 +size 6857549120 diff --git a/README.md b/README.md new file mode 100644 index 0000000..06a852d --- /dev/null +++ b/README.md @@ -0,0 +1,76 @@ +--- +base_model: TencentARC/LLaMA-Pro-8B-Instruct +language: +- en +library_name: transformers +license: llama2 +quantized_by: mradermacher +--- +## About + + + + + + +weighted/imatrix quants of https://huggingface.co/TencentARC/LLaMA-Pro-8B-Instruct + + +static quants are available at https://huggingface.co/mradermacher/LLaMA-Pro-8B-Instruct-GGUF +## Usage + +If you are unsure how to use GGUF files, refer to one of [TheBloke's +READMEs](https://huggingface.co/TheBloke/KafkaLM-70B-German-V0.1-GGUF) for +more details, including on how to concatenate multi-part files. + +## Provided Quants + +(sorted by size, not necessarily quality. IQ-quants are often preferable over similar sized non-IQ quants) + +| Link | Type | Size/GB | Notes | +|:-----|:-----|--------:|:------| +| [GGUF](https://huggingface.co/mradermacher/LLaMA-Pro-8B-Instruct-i1-GGUF/resolve/main/LLaMA-Pro-8B-Instruct.i1-IQ1_S.gguf) | i1-IQ1_S | 2.0 | for the desperate | +| [GGUF](https://huggingface.co/mradermacher/LLaMA-Pro-8B-Instruct-i1-GGUF/resolve/main/LLaMA-Pro-8B-Instruct.i1-IQ1_M.gguf) | i1-IQ1_M | 2.1 | mostly desperate | +| [GGUF](https://huggingface.co/mradermacher/LLaMA-Pro-8B-Instruct-i1-GGUF/resolve/main/LLaMA-Pro-8B-Instruct.i1-IQ2_XXS.gguf) | i1-IQ2_XXS | 2.4 | | +| [GGUF](https://huggingface.co/mradermacher/LLaMA-Pro-8B-Instruct-i1-GGUF/resolve/main/LLaMA-Pro-8B-Instruct.i1-IQ2_XS.gguf) | i1-IQ2_XS | 2.6 | | +| [GGUF](https://huggingface.co/mradermacher/LLaMA-Pro-8B-Instruct-i1-GGUF/resolve/main/LLaMA-Pro-8B-Instruct.i1-IQ2_S.gguf) | i1-IQ2_S | 2.8 | | +| [GGUF](https://huggingface.co/mradermacher/LLaMA-Pro-8B-Instruct-i1-GGUF/resolve/main/LLaMA-Pro-8B-Instruct.i1-IQ2_M.gguf) | i1-IQ2_M | 3.0 | | +| [GGUF](https://huggingface.co/mradermacher/LLaMA-Pro-8B-Instruct-i1-GGUF/resolve/main/LLaMA-Pro-8B-Instruct.i1-Q2_K.gguf) | i1-Q2_K | 3.2 | IQ3_XXS probably better | +| [GGUF](https://huggingface.co/mradermacher/LLaMA-Pro-8B-Instruct-i1-GGUF/resolve/main/LLaMA-Pro-8B-Instruct.i1-IQ3_XXS.gguf) | i1-IQ3_XXS | 3.3 | lower quality | +| [GGUF](https://huggingface.co/mradermacher/LLaMA-Pro-8B-Instruct-i1-GGUF/resolve/main/LLaMA-Pro-8B-Instruct.i1-IQ3_XS.gguf) | i1-IQ3_XS | 3.6 | | +| [GGUF](https://huggingface.co/mradermacher/LLaMA-Pro-8B-Instruct-i1-GGUF/resolve/main/LLaMA-Pro-8B-Instruct.i1-IQ3_S.gguf) | i1-IQ3_S | 3.7 | beats Q3_K* | +| [GGUF](https://huggingface.co/mradermacher/LLaMA-Pro-8B-Instruct-i1-GGUF/resolve/main/LLaMA-Pro-8B-Instruct.i1-Q3_K_S.gguf) | i1-Q3_K_S | 3.7 | IQ3_XS probably better | +| [GGUF](https://huggingface.co/mradermacher/LLaMA-Pro-8B-Instruct-i1-GGUF/resolve/main/LLaMA-Pro-8B-Instruct.i1-IQ3_M.gguf) | i1-IQ3_M | 4.0 | | +| [GGUF](https://huggingface.co/mradermacher/LLaMA-Pro-8B-Instruct-i1-GGUF/resolve/main/LLaMA-Pro-8B-Instruct.i1-Q3_K_M.gguf) | i1-Q3_K_M | 4.2 | IQ3_S probably better | +| [GGUF](https://huggingface.co/mradermacher/LLaMA-Pro-8B-Instruct-i1-GGUF/resolve/main/LLaMA-Pro-8B-Instruct.i1-Q3_K_L.gguf) | i1-Q3_K_L | 4.6 | IQ3_M probably better | +| [GGUF](https://huggingface.co/mradermacher/LLaMA-Pro-8B-Instruct-i1-GGUF/resolve/main/LLaMA-Pro-8B-Instruct.i1-IQ4_XS.gguf) | i1-IQ4_XS | 4.6 | | +| [GGUF](https://huggingface.co/mradermacher/LLaMA-Pro-8B-Instruct-i1-GGUF/resolve/main/LLaMA-Pro-8B-Instruct.i1-Q4_0_4_4.gguf) | i1-Q4_0_4_4 | 4.8 | fast on arm, low quality | +| [GGUF](https://huggingface.co/mradermacher/LLaMA-Pro-8B-Instruct-i1-GGUF/resolve/main/LLaMA-Pro-8B-Instruct.i1-Q4_0_4_8.gguf) | i1-Q4_0_4_8 | 4.8 | fast on arm+i8mm, low quality | +| [GGUF](https://huggingface.co/mradermacher/LLaMA-Pro-8B-Instruct-i1-GGUF/resolve/main/LLaMA-Pro-8B-Instruct.i1-Q4_0_8_8.gguf) | i1-Q4_0_8_8 | 4.8 | fast on arm+sve, low quality | +| [GGUF](https://huggingface.co/mradermacher/LLaMA-Pro-8B-Instruct-i1-GGUF/resolve/main/LLaMA-Pro-8B-Instruct.i1-Q4_0.gguf) | i1-Q4_0 | 4.9 | fast, low quality | +| [GGUF](https://huggingface.co/mradermacher/LLaMA-Pro-8B-Instruct-i1-GGUF/resolve/main/LLaMA-Pro-8B-Instruct.i1-Q4_K_S.gguf) | i1-Q4_K_S | 4.9 | optimal size/speed/quality | +| [GGUF](https://huggingface.co/mradermacher/LLaMA-Pro-8B-Instruct-i1-GGUF/resolve/main/LLaMA-Pro-8B-Instruct.i1-Q4_K_M.gguf) | i1-Q4_K_M | 5.2 | fast, recommended | +| [GGUF](https://huggingface.co/mradermacher/LLaMA-Pro-8B-Instruct-i1-GGUF/resolve/main/LLaMA-Pro-8B-Instruct.i1-Q5_K_S.gguf) | i1-Q5_K_S | 5.9 | | +| [GGUF](https://huggingface.co/mradermacher/LLaMA-Pro-8B-Instruct-i1-GGUF/resolve/main/LLaMA-Pro-8B-Instruct.i1-Q5_K_M.gguf) | i1-Q5_K_M | 6.0 | | +| [GGUF](https://huggingface.co/mradermacher/LLaMA-Pro-8B-Instruct-i1-GGUF/resolve/main/LLaMA-Pro-8B-Instruct.i1-Q6_K.gguf) | i1-Q6_K | 7.0 | practically like static Q6_K | + +Here is a handy graph by ikawrakow comparing some lower-quality quant +types (lower is better): + +![image.png](https://www.nethype.de/huggingface_embed/quantpplgraph.png) + +And here are Artefact2's thoughts on the matter: +https://gist.github.com/Artefact2/b5f810600771265fc1e39442288e8ec9 + +## FAQ / Model Request + +See https://huggingface.co/mradermacher/model_requests for some answers to +questions you might have and/or if you want some other model quantized. + +## Thanks + +I thank my company, [nethype GmbH](https://www.nethype.de/), for letting +me use its servers and providing upgrades to my workstation to enable +this work in my free time. Additional thanks to [@nicoboss](https://huggingface.co/nicoboss) for giving me access to his private supercomputer, enabling me to provide many more imatrix quants, at much higher quality, than I would otherwise be able to. + + diff --git a/imatrix.dat b/imatrix.dat new file mode 100644 index 0000000..374d03a --- /dev/null +++ b/imatrix.dat @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d4b6d2207479f1d1e3bd3ffba3d93c981c99e8e6765e25a573f1809a1c2839f6 +size 5702725