diff --git a/.gitattributes b/.gitattributes index b7722c6..605a146 100644 --- a/.gitattributes +++ b/.gitattributes @@ -46,3 +46,16 @@ Meta-Llama-3.1-8B-Instruct-Q5_K_S.gguf filter=lfs diff=lfs merge=lfs -text Meta-Llama-3.1-8B-Instruct-Q6_K.gguf filter=lfs diff=lfs merge=lfs -text Meta-Llama-3.1-8B-Instruct-Q8_0.gguf filter=lfs diff=lfs merge=lfs -text Meta-Llama-3.1-8B-Instruct-f16.gguf filter=lfs diff=lfs merge=lfs -text +Llama-3.1-8B-Instruct-Q2_K.gguf filter=lfs diff=lfs merge=lfs -text +Llama-3.1-8B-Instruct-Q3_K_L.gguf filter=lfs diff=lfs merge=lfs -text +Llama-3.1-8B-Instruct-Q3_K_M.gguf filter=lfs diff=lfs merge=lfs -text +Llama-3.1-8B-Instruct-Q3_K_S.gguf filter=lfs diff=lfs merge=lfs -text +Llama-3.1-8B-Instruct-Q4_0.gguf filter=lfs diff=lfs merge=lfs -text +Llama-3.1-8B-Instruct-Q4_K_M.gguf filter=lfs diff=lfs merge=lfs -text +Llama-3.1-8B-Instruct-Q4_K_S.gguf filter=lfs diff=lfs merge=lfs -text +Llama-3.1-8B-Instruct-Q5_0.gguf filter=lfs diff=lfs merge=lfs -text +Llama-3.1-8B-Instruct-Q5_K_M.gguf filter=lfs diff=lfs merge=lfs -text +Llama-3.1-8B-Instruct-Q5_K_S.gguf filter=lfs diff=lfs merge=lfs -text +Llama-3.1-8B-Instruct-Q6_K.gguf filter=lfs diff=lfs merge=lfs -text +Llama-3.1-8B-Instruct-Q8_0.gguf filter=lfs diff=lfs merge=lfs -text +Llama-3.1-8B-Instruct-f16.gguf filter=lfs diff=lfs merge=lfs -text diff --git a/Llama-3.1-8B-Instruct-Q2_K.gguf b/Llama-3.1-8B-Instruct-Q2_K.gguf new file mode 100644 index 0000000..81cc653 --- /dev/null +++ b/Llama-3.1-8B-Instruct-Q2_K.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bd52c261c8b48d964542130cee85cc027f844dc3b5b17064942e00713abb4eee +size 3179136384 diff --git a/Llama-3.1-8B-Instruct-Q3_K_L.gguf b/Llama-3.1-8B-Instruct-Q3_K_L.gguf new file mode 100644 index 0000000..4a92f99 --- /dev/null +++ b/Llama-3.1-8B-Instruct-Q3_K_L.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:20aa85b66c8056cc2db641260651554ffee6c7f93410edbdaf0e8e46b968cefd +size 4321961344 diff --git a/Llama-3.1-8B-Instruct-Q3_K_M.gguf b/Llama-3.1-8B-Instruct-Q3_K_M.gguf new file mode 100644 index 0000000..2a16470 --- /dev/null +++ b/Llama-3.1-8B-Instruct-Q3_K_M.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:024edce38ba06f957af68a9e4ebf0691e1a897196ed5c067434bb488dc26d800 +size 4018922880 diff --git a/Llama-3.1-8B-Instruct-Q3_K_S.gguf b/Llama-3.1-8B-Instruct-Q3_K_S.gguf new file mode 100644 index 0000000..d945569 --- /dev/null +++ b/Llama-3.1-8B-Instruct-Q3_K_S.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:843e53b71ce4dd4526d08ecff24c6bc6fcadb260dcc94bf1e75d43d337af9df2 +size 3664504192 diff --git a/Llama-3.1-8B-Instruct-Q4_0.gguf b/Llama-3.1-8B-Instruct-Q4_0.gguf new file mode 100644 index 0000000..bcbedd4 --- /dev/null +++ b/Llama-3.1-8B-Instruct-Q4_0.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:01e1a5284dc2b7c202544a3e6ff64b08265563aae1a76b1c8938f5edc266e325 +size 4661216640 diff --git a/Llama-3.1-8B-Instruct-Q4_K_M.gguf b/Llama-3.1-8B-Instruct-Q4_K_M.gguf new file mode 100644 index 0000000..8954c58 --- /dev/null +++ b/Llama-3.1-8B-Instruct-Q4_K_M.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:599024f21235045ea4ad7aecc5d46e794920e201d669328f577b1971cf98c904 +size 4920739200 diff --git a/Llama-3.1-8B-Instruct-Q4_K_S.gguf b/Llama-3.1-8B-Instruct-Q4_K_S.gguf new file mode 100644 index 0000000..d5b05d7 --- /dev/null +++ b/Llama-3.1-8B-Instruct-Q4_K_S.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8f04f87f357c452540d8bf63e73f7145ed46281f9ffed434316edcc9b9f62ad2 +size 4692673920 diff --git a/Llama-3.1-8B-Instruct-Q5_0.gguf b/Llama-3.1-8B-Instruct-Q5_0.gguf new file mode 100644 index 0000000..4d57b2b --- /dev/null +++ b/Llama-3.1-8B-Instruct-Q5_0.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f09d287ea0dbbae61252d7a0447dba8528d9e087dfa0d5b2568ad0ddc59596b0 +size 5599298944 diff --git a/Llama-3.1-8B-Instruct-Q5_K_M.gguf b/Llama-3.1-8B-Instruct-Q5_K_M.gguf new file mode 100644 index 0000000..e6c3d8c --- /dev/null +++ b/Llama-3.1-8B-Instruct-Q5_K_M.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1240ea85e6c113d20e858649e0a8325f2187683b0c69075be6d184320d9d5c79 +size 5732992384 diff --git a/Llama-3.1-8B-Instruct-Q5_K_S.gguf b/Llama-3.1-8B-Instruct-Q5_K_S.gguf new file mode 100644 index 0000000..2360674 --- /dev/null +++ b/Llama-3.1-8B-Instruct-Q5_K_S.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6c80108e113b6d682dcb18c8487b52c3178690465e53481586d22146293613a8 +size 5599298944 diff --git a/Llama-3.1-8B-Instruct-Q6_K.gguf b/Llama-3.1-8B-Instruct-Q6_K.gguf new file mode 100644 index 0000000..fdfce47 --- /dev/null +++ b/Llama-3.1-8B-Instruct-Q6_K.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f6ac847bb62c2e038b6c638c8eff6fa6416737096d7350619cbb75d29a5d61ba +size 6596011392 diff --git a/Llama-3.1-8B-Instruct-Q8_0.gguf b/Llama-3.1-8B-Instruct-Q8_0.gguf new file mode 100644 index 0000000..6b1553f --- /dev/null +++ b/Llama-3.1-8B-Instruct-Q8_0.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:798c1d12513bf2b2c9762c2ff5b9136de08fbbac0b498fe8b83d4f00d73d4044 +size 8540775808 diff --git a/Llama-3.1-8B-Instruct-f16.gguf b/Llama-3.1-8B-Instruct-f16.gguf new file mode 100644 index 0000000..2d258a8 --- /dev/null +++ b/Llama-3.1-8B-Instruct-f16.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:031c5d9ae1ad8cf676f8498ef539418c0d1f2b1b9c9d33c8f9207b83218c89b3 +size 16068896128 diff --git a/Meta-Llama-3.1-8B-Instruct-Q2_K.gguf b/Meta-Llama-3.1-8B-Instruct-Q2_K.gguf deleted file mode 100644 index ab4c41a..0000000 --- a/Meta-Llama-3.1-8B-Instruct-Q2_K.gguf +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:978c23cad76854c8bf944a819159697215575a72a8d70910b41e9effbd7badfb -size 3179131552 diff --git a/Meta-Llama-3.1-8B-Instruct-Q3_K_L.gguf b/Meta-Llama-3.1-8B-Instruct-Q3_K_L.gguf deleted file mode 100644 index d59fa31..0000000 --- a/Meta-Llama-3.1-8B-Instruct-Q3_K_L.gguf +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:46bb86256326b4fea38c4adf8bd70b4e0ad5551db43ef9fe0e5489411ca0dc1f -size 4321956512 diff --git a/Meta-Llama-3.1-8B-Instruct-Q3_K_M.gguf b/Meta-Llama-3.1-8B-Instruct-Q3_K_M.gguf deleted file mode 100644 index 7c9f1d2..0000000 --- a/Meta-Llama-3.1-8B-Instruct-Q3_K_M.gguf +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:1577fefd7a7d843f89b25d32849e3fa2e19a5373e620bce504d2dbc57b73f1cf -size 4018918048 diff --git a/Meta-Llama-3.1-8B-Instruct-Q3_K_S.gguf b/Meta-Llama-3.1-8B-Instruct-Q3_K_S.gguf deleted file mode 100644 index f553f2a..0000000 --- a/Meta-Llama-3.1-8B-Instruct-Q3_K_S.gguf +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:b74d716226d30c2516d26bbd4cff01f951433bfe717e1ef4b61899fbaf7a8152 -size 3664499360 diff --git a/Meta-Llama-3.1-8B-Instruct-Q4_0.gguf b/Meta-Llama-3.1-8B-Instruct-Q4_0.gguf deleted file mode 100644 index 5b6ba96..0000000 --- a/Meta-Llama-3.1-8B-Instruct-Q4_0.gguf +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:323f9cc1282b3d5bb817a0af2f3cfd0a6a68496d02ae6d420f5b14ed688ceb3f -size 4661211808 diff --git a/Meta-Llama-3.1-8B-Instruct-Q4_K_M.gguf b/Meta-Llama-3.1-8B-Instruct-Q4_K_M.gguf deleted file mode 100644 index e217dc7..0000000 --- a/Meta-Llama-3.1-8B-Instruct-Q4_K_M.gguf +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:415df20ac06158f3b473bf6ea3aa666e052ed55e6e6ff6b0dbb9db7af4180d36 -size 4920734368 diff --git a/Meta-Llama-3.1-8B-Instruct-Q4_K_S.gguf b/Meta-Llama-3.1-8B-Instruct-Q4_K_S.gguf deleted file mode 100644 index aa0fc38..0000000 --- a/Meta-Llama-3.1-8B-Instruct-Q4_K_S.gguf +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:33a8ba1673c3e6fdca8397a8abd35af58cde851e471c30fc4a621f80274f3f92 -size 4692669088 diff --git a/Meta-Llama-3.1-8B-Instruct-Q5_0.gguf b/Meta-Llama-3.1-8B-Instruct-Q5_0.gguf deleted file mode 100644 index 0100c82..0000000 --- a/Meta-Llama-3.1-8B-Instruct-Q5_0.gguf +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:5b053deabe91246648fabf95b972d6a64e4077e432a288a50629aefe2fcfa719 -size 5599294112 diff --git a/Meta-Llama-3.1-8B-Instruct-Q5_K_M.gguf b/Meta-Llama-3.1-8B-Instruct-Q5_K_M.gguf deleted file mode 100644 index 3a9b901..0000000 --- a/Meta-Llama-3.1-8B-Instruct-Q5_K_M.gguf +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:b4e75ee4ff8356eb25ad9f5055cf527e7d2654c005dde8fb5481db50e5164e0f -size 5732987552 diff --git a/Meta-Llama-3.1-8B-Instruct-Q5_K_S.gguf b/Meta-Llama-3.1-8B-Instruct-Q5_K_S.gguf deleted file mode 100644 index 998a13d..0000000 --- a/Meta-Llama-3.1-8B-Instruct-Q5_K_S.gguf +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:61a6be00c21161cf45c1f0172a0fdbf4a24f1cb58c84593c66c2db74527de2b3 -size 5599294112 diff --git a/Meta-Llama-3.1-8B-Instruct-Q6_K.gguf b/Meta-Llama-3.1-8B-Instruct-Q6_K.gguf deleted file mode 100644 index 327cfa8..0000000 --- a/Meta-Llama-3.1-8B-Instruct-Q6_K.gguf +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:0b49f782d79e5d33b203c5fca60140586ae3abeec80b894bf8db40f594786ece -size 6596006560 diff --git a/Meta-Llama-3.1-8B-Instruct-Q8_0.gguf b/Meta-Llama-3.1-8B-Instruct-Q8_0.gguf deleted file mode 100644 index 63e9446..0000000 --- a/Meta-Llama-3.1-8B-Instruct-Q8_0.gguf +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:91818e082e2d87acf79032cb0c9d8e4e4878b1ee38ba68f09e120d47b3aeb04c -size 8540770976 diff --git a/Meta-Llama-3.1-8B-Instruct-f16.gguf b/Meta-Llama-3.1-8B-Instruct-f16.gguf deleted file mode 100644 index 38ec6cd..0000000 --- a/Meta-Llama-3.1-8B-Instruct-f16.gguf +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:82ff778433e6590189788b54c00aba834d6eb167808ec32ae659964392a07230 -size 16068891296 diff --git a/README.md b/README.md index 9892ad1..d6142b4 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,7 @@ tags: ## Run with LlamaEdge -- LlamaEdge version: [v0.12.4](https://github.com/LlamaEdge/LlamaEdge/releases/tag/0.12.4) and above +- LlamaEdge version: [v0.16.5](https://github.com/LlamaEdge/LlamaEdge/releases/tag/0.16.5) and above - Prompt template @@ -86,7 +86,7 @@ tags: - Chat ```bash - wasmedge --dir .:. --nn-preload default:GGML:AUTO:Meta-Llama-3.1-8B-Instruct-Q5_K_M.gguf \ + wasmedge --dir .:. --nn-preload default:GGML:AUTO:Llama-3.1-8B-Instruct-Q5_K_M.gguf \ llama-api-server.wasm \ --prompt-template llama-3-chat \ --ctx-size 128000 \ @@ -96,7 +96,7 @@ tags: - Tool use ```bash - wasmedge --dir .:. --nn-preload default:GGML:AUTO:Meta-Llama-3.1-8B-Instruct-Q5_K_M.gguf \ + wasmedge --dir .:. --nn-preload default:GGML:AUTO:Llama-3.1-8B-Instruct-Q5_K_M.gguf \ llama-api-server.wasm \ --prompt-template llama-3-tool \ --ctx-size 128000 \ @@ -106,7 +106,7 @@ tags: - Run as LlamaEdge command app ```bash - wasmedge --dir .:. --nn-preload default:GGML:AUTO:Meta-Llama-3.1-8B-Instruct-Q5_K_M.gguf \ + wasmedge --dir .:. --nn-preload default:GGML:AUTO:Llama-3.1-8B-Instruct-Q5_K_M.gguf \ llama-chat.wasm \ --prompt-template llama-3-chat \ --ctx-size 128000 @@ -116,18 +116,18 @@ tags: | Name | Quant method | Bits | Size | Use case | | ---- | ---- | ---- | ---- | ----- | -| [Meta-Llama-3.1-8B-Instruct-Q2_K.gguf](https://huggingface.co/second-state/Meta-Llama-3.1-8B-Instruct-GGUF/blob/main/Meta-Llama-3.1-8B-Instruct-Q2_K.gguf) | Q2_K | 2 | 3.18 GB| smallest, significant quality loss - not recommended for most purposes | -| [Meta-Llama-3.1-8B-Instruct-Q3_K_L.gguf](https://huggingface.co/second-state/Meta-Llama-3.1-8B-Instruct-GGUF/blob/main/Meta-Llama-3.1-8B-Instruct-Q3_K_L.gguf) | Q3_K_L | 3 | 4.32 GB| small, substantial quality loss | -| [Meta-Llama-3.1-8B-Instruct-Q3_K_M.gguf](https://huggingface.co/second-state/Meta-Llama-3.1-8B-Instruct-GGUF/blob/main/Meta-Llama-3.1-8B-Instruct-Q3_K_M.gguf) | Q3_K_M | 3 | 4.02 GB| very small, high quality loss | -| [Meta-Llama-3.1-8B-Instruct-Q3_K_S.gguf](https://huggingface.co/second-state/Meta-Llama-3.1-8B-Instruct-GGUF/blob/main/Meta-Llama-3.1-8B-Instruct-Q3_K_S.gguf) | Q3_K_S | 3 | 3.66 GB| very small, high quality loss | -| [Meta-Llama-3.1-8B-Instruct-Q4_0.gguf](https://huggingface.co/second-state/Meta-Llama-3.1-8B-Instruct-GGUF/blob/main/Meta-Llama-3.1-8B-Instruct-Q4_0.gguf) | Q4_0 | 4 | 4.66 GB| legacy; small, very high quality loss - prefer using Q3_K_M | -| [Meta-Llama-3.1-8B-Instruct-Q4_K_M.gguf](https://huggingface.co/second-state/Meta-Llama-3.1-8B-Instruct-GGUF/blob/main/Meta-Llama-3.1-8B-Instruct-Q4_K_M.gguf) | Q4_K_M | 4 | 4.92 GB| medium, balanced quality - recommended | -| [Meta-Llama-3.1-8B-Instruct-Q4_K_S.gguf](https://huggingface.co/second-state/Meta-Llama-3.1-8B-Instruct-GGUF/blob/main/Meta-Llama-3.1-8B-Instruct-Q4_K_S.gguf) | Q4_K_S | 4 | 4.69 GB| small, greater quality loss | -| [Meta-Llama-3.1-8B-Instruct-Q5_0.gguf](https://huggingface.co/second-state/Meta-Llama-3.1-8B-Instruct-GGUF/blob/main/Meta-Llama-3.1-8B-Instruct-Q5_0.gguf) | Q5_0 | 5 | 5.6 GB| legacy; medium, balanced quality - prefer using Q4_K_M | -| [Meta-Llama-3.1-8B-Instruct-Q5_K_M.gguf](https://huggingface.co/second-state/Meta-Llama-3.1-8B-Instruct-GGUF/blob/main/Meta-Llama-3.1-8B-Instruct-Q5_K_M.gguf) | Q5_K_M | 5 | 5.73 GB| large, very low quality loss - recommended | -| [Meta-Llama-3.1-8B-Instruct-Q5_K_S.gguf](https://huggingface.co/second-state/Meta-Llama-3.1-8B-Instruct-GGUF/blob/main/Meta-Llama-3.1-8B-Instruct-Q5_K_S.gguf) | Q5_K_S | 5 | 5.6 GB| large, low quality loss - recommended | -| [Meta-Llama-3.1-8B-Instruct-Q6_K.gguf](https://huggingface.co/second-state/Meta-Llama-3.1-8B-Instruct-GGUF/blob/main/Meta-Llama-3.1-8B-Instruct-Q6_K.gguf) | Q6_K | 6 | 6.6 GB| very large, extremely low quality loss | -| [Meta-Llama-3.1-8B-Instruct-Q8_0.gguf](https://huggingface.co/second-state/Meta-Llama-3.1-8B-Instruct-GGUF/blob/main/Meta-Llama-3.1-8B-Instruct-Q8_0.gguf) | Q8_0 | 8 | 8.54 GB| very large, extremely low quality loss - not recommended | -| [Meta-Llama-3.1-8B-Instruct-f16.gguf](https://huggingface.co/second-state/Meta-Llama-3.1-8B-Instruct-GGUF/blob/main/Meta-Llama-3.1-8B-Instruct-f16.gguf) | f16 | 16 | 16.1 GB| | +| [Llama-3.1-8B-Instruct-Q2_K.gguf](https://huggingface.co/second-state/Meta-Llama-3.1-8B-Instruct-GGUF/blob/main/Llama-3.1-8B-Instruct-Q2_K.gguf) | Q2_K | 2 | 3.18 GB| smallest, significant quality loss - not recommended for most purposes | +| [Llama-3.1-8B-Instruct-Q3_K_L.gguf](https://huggingface.co/second-state/Meta-Llama-3.1-8B-Instruct-GGUF/blob/main/Llama-3.1-8B-Instruct-Q3_K_L.gguf) | Q3_K_L | 3 | 4.32 GB| small, substantial quality loss | +| [Llama-3.1-8B-Instruct-Q3_K_M.gguf](https://huggingface.co/second-state/Meta-Llama-3.1-8B-Instruct-GGUF/blob/main/Llama-3.1-8B-Instruct-Q3_K_M.gguf) | Q3_K_M | 3 | 4.02 GB| very small, high quality loss | +| [Llama-3.1-8B-Instruct-Q3_K_S.gguf](https://huggingface.co/second-state/Meta-Llama-3.1-8B-Instruct-GGUF/blob/main/Llama-3.1-8B-Instruct-Q3_K_S.gguf) | Q3_K_S | 3 | 3.66 GB| very small, high quality loss | +| [Llama-3.1-8B-Instruct-Q4_0.gguf](https://huggingface.co/second-state/Meta-Llama-3.1-8B-Instruct-GGUF/blob/main/Llama-3.1-8B-Instruct-Q4_0.gguf) | Q4_0 | 4 | 4.66 GB| legacy; small, very high quality loss - prefer using Q3_K_M | +| [Llama-3.1-8B-Instruct-Q4_K_M.gguf](https://huggingface.co/second-state/Meta-Llama-3.1-8B-Instruct-GGUF/blob/main/Llama-3.1-8B-Instruct-Q4_K_M.gguf) | Q4_K_M | 4 | 4.92 GB| medium, balanced quality - recommended | +| [Llama-3.1-8B-Instruct-Q4_K_S.gguf](https://huggingface.co/second-state/Meta-Llama-3.1-8B-Instruct-GGUF/blob/main/Llama-3.1-8B-Instruct-Q4_K_S.gguf) | Q4_K_S | 4 | 4.69 GB| small, greater quality loss | +| [Llama-3.1-8B-Instruct-Q5_0.gguf](https://huggingface.co/second-state/Meta-Llama-3.1-8B-Instruct-GGUF/blob/main/Llama-3.1-8B-Instruct-Q5_0.gguf) | Q5_0 | 5 | 5.6 GB| legacy; medium, balanced quality - prefer using Q4_K_M | +| [Llama-3.1-8B-Instruct-Q5_K_M.gguf](https://huggingface.co/second-state/Meta-Llama-3.1-8B-Instruct-GGUF/blob/main/Llama-3.1-8B-Instruct-Q5_K_M.gguf) | Q5_K_M | 5 | 5.73 GB| large, very low quality loss - recommended | +| [Llama-3.1-8B-Instruct-Q5_K_S.gguf](https://huggingface.co/second-state/Meta-Llama-3.1-8B-Instruct-GGUF/blob/main/Llama-3.1-8B-Instruct-Q5_K_S.gguf) | Q5_K_S | 5 | 5.6 GB| large, low quality loss - recommended | +| [Llama-3.1-8B-Instruct-Q6_K.gguf](https://huggingface.co/second-state/Meta-Llama-3.1-8B-Instruct-GGUF/blob/main/Llama-3.1-8B-Instruct-Q6_K.gguf) | Q6_K | 6 | 6.6 GB| very large, extremely low quality loss | +| [Llama-3.1-8B-Instruct-Q8_0.gguf](https://huggingface.co/second-state/Meta-Llama-3.1-8B-Instruct-GGUF/blob/main/Llama-3.1-8B-Instruct-Q8_0.gguf) | Q8_0 | 8 | 8.54 GB| very large, extremely low quality loss - not recommended | +| [Llama-3.1-8B-Instruct-f16.gguf](https://huggingface.co/second-state/Meta-Llama-3.1-8B-Instruct-GGUF/blob/main/Llama-3.1-8B-Instruct-f16.gguf) | f16 | 16 | 16.1 GB| | -*Quantized with llama.cpp b3445.* +*Quantized with llama.cpp b4466.*