From e95597bf834f09a5c3561d5cbf76c6007b2e69ee Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Tue, 15 Sep 2026 08:24:15 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: second-state/DeepHermes-3-Llama-3-8B-Preview-GGUF Source: Original Platform --- .gitattributes | 48 ++++++++ DeepHermes-3-Llama-3-8B-Preview-Q2_K.gguf | 3 + DeepHermes-3-Llama-3-8B-Preview-Q3_K_L.gguf | 3 + DeepHermes-3-Llama-3-8B-Preview-Q3_K_M.gguf | 3 + DeepHermes-3-Llama-3-8B-Preview-Q3_K_S.gguf | 3 + DeepHermes-3-Llama-3-8B-Preview-Q4_0.gguf | 3 + DeepHermes-3-Llama-3-8B-Preview-Q4_K_M.gguf | 3 + DeepHermes-3-Llama-3-8B-Preview-Q4_K_S.gguf | 3 + DeepHermes-3-Llama-3-8B-Preview-Q5_0.gguf | 3 + DeepHermes-3-Llama-3-8B-Preview-Q5_K_M.gguf | 3 + DeepHermes-3-Llama-3-8B-Preview-Q5_K_S.gguf | 3 + DeepHermes-3-Llama-3-8B-Preview-Q6_K.gguf | 3 + DeepHermes-3-Llama-3-8B-Preview-Q8_0.gguf | 3 + DeepHermes-3-Llama-3-8B-Preview-f16.gguf | 3 + README.md | 123 ++++++++++++++++++++ config.json | 38 ++++++ 16 files changed, 248 insertions(+) create mode 100644 .gitattributes create mode 100644 DeepHermes-3-Llama-3-8B-Preview-Q2_K.gguf create mode 100644 DeepHermes-3-Llama-3-8B-Preview-Q3_K_L.gguf create mode 100644 DeepHermes-3-Llama-3-8B-Preview-Q3_K_M.gguf create mode 100644 DeepHermes-3-Llama-3-8B-Preview-Q3_K_S.gguf create mode 100644 DeepHermes-3-Llama-3-8B-Preview-Q4_0.gguf create mode 100644 DeepHermes-3-Llama-3-8B-Preview-Q4_K_M.gguf create mode 100644 DeepHermes-3-Llama-3-8B-Preview-Q4_K_S.gguf create mode 100644 DeepHermes-3-Llama-3-8B-Preview-Q5_0.gguf create mode 100644 DeepHermes-3-Llama-3-8B-Preview-Q5_K_M.gguf create mode 100644 DeepHermes-3-Llama-3-8B-Preview-Q5_K_S.gguf create mode 100644 DeepHermes-3-Llama-3-8B-Preview-Q6_K.gguf create mode 100644 DeepHermes-3-Llama-3-8B-Preview-Q8_0.gguf create mode 100644 DeepHermes-3-Llama-3-8B-Preview-f16.gguf create mode 100644 README.md create mode 100644 config.json diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..1c6c8cb --- /dev/null +++ b/.gitattributes @@ -0,0 +1,48 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +DeepHermes-3-Llama-3-8B-Preview-Q2_K.gguf filter=lfs diff=lfs merge=lfs -text +DeepHermes-3-Llama-3-8B-Preview-Q3_K_L.gguf filter=lfs diff=lfs merge=lfs -text +DeepHermes-3-Llama-3-8B-Preview-Q3_K_M.gguf filter=lfs diff=lfs merge=lfs -text +DeepHermes-3-Llama-3-8B-Preview-Q3_K_S.gguf filter=lfs diff=lfs merge=lfs -text +DeepHermes-3-Llama-3-8B-Preview-Q4_0.gguf filter=lfs diff=lfs merge=lfs -text +DeepHermes-3-Llama-3-8B-Preview-Q4_K_M.gguf filter=lfs diff=lfs merge=lfs -text +DeepHermes-3-Llama-3-8B-Preview-Q4_K_S.gguf filter=lfs diff=lfs merge=lfs -text +DeepHermes-3-Llama-3-8B-Preview-Q5_0.gguf filter=lfs diff=lfs merge=lfs -text +DeepHermes-3-Llama-3-8B-Preview-Q5_K_M.gguf filter=lfs diff=lfs merge=lfs -text +DeepHermes-3-Llama-3-8B-Preview-Q5_K_S.gguf filter=lfs diff=lfs merge=lfs -text +DeepHermes-3-Llama-3-8B-Preview-Q6_K.gguf filter=lfs diff=lfs merge=lfs -text +DeepHermes-3-Llama-3-8B-Preview-Q8_0.gguf filter=lfs diff=lfs merge=lfs -text +DeepHermes-3-Llama-3-8B-Preview-f16.gguf filter=lfs diff=lfs merge=lfs -text diff --git a/DeepHermes-3-Llama-3-8B-Preview-Q2_K.gguf b/DeepHermes-3-Llama-3-8B-Preview-Q2_K.gguf new file mode 100644 index 0000000..32ebd32 --- /dev/null +++ b/DeepHermes-3-Llama-3-8B-Preview-Q2_K.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:82a325a034b1e182938ae2a536b14febc88032958a2eb45d497d5e451ac65d39 +size 3179137664 diff --git a/DeepHermes-3-Llama-3-8B-Preview-Q3_K_L.gguf b/DeepHermes-3-Llama-3-8B-Preview-Q3_K_L.gguf new file mode 100644 index 0000000..e873a9b --- /dev/null +++ b/DeepHermes-3-Llama-3-8B-Preview-Q3_K_L.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a2c6628ddf9f614cbc55d9af08c0d8f1d02ff7003d62d734b7e3295e6d9f0701 +size 4321962624 diff --git a/DeepHermes-3-Llama-3-8B-Preview-Q3_K_M.gguf b/DeepHermes-3-Llama-3-8B-Preview-Q3_K_M.gguf new file mode 100644 index 0000000..eed4083 --- /dev/null +++ b/DeepHermes-3-Llama-3-8B-Preview-Q3_K_M.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7586500237fc8c5d8f2972bd928eeed4df15ceafba51984f97697fefb3785da +size 4018924160 diff --git a/DeepHermes-3-Llama-3-8B-Preview-Q3_K_S.gguf b/DeepHermes-3-Llama-3-8B-Preview-Q3_K_S.gguf new file mode 100644 index 0000000..87638f5 --- /dev/null +++ b/DeepHermes-3-Llama-3-8B-Preview-Q3_K_S.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bf86d886ad20c42c491d6a409545bc61afc41b613dc165d48ae2bde3b59ba387 +size 3664505472 diff --git a/DeepHermes-3-Llama-3-8B-Preview-Q4_0.gguf b/DeepHermes-3-Llama-3-8B-Preview-Q4_0.gguf new file mode 100644 index 0000000..3bb3432 --- /dev/null +++ b/DeepHermes-3-Llama-3-8B-Preview-Q4_0.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bd32e18873f21c665e60f466e4a49159c34b2f1bcd66e7c0a90999027e67a212 +size 4661217920 diff --git a/DeepHermes-3-Llama-3-8B-Preview-Q4_K_M.gguf b/DeepHermes-3-Llama-3-8B-Preview-Q4_K_M.gguf new file mode 100644 index 0000000..f118009 --- /dev/null +++ b/DeepHermes-3-Llama-3-8B-Preview-Q4_K_M.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:11870b66a8e1faa9020b8a26bdf1b9b7bcf70ff8f28b8e5a95cd5ef202e807c1 +size 4920740480 diff --git a/DeepHermes-3-Llama-3-8B-Preview-Q4_K_S.gguf b/DeepHermes-3-Llama-3-8B-Preview-Q4_K_S.gguf new file mode 100644 index 0000000..94153c0 --- /dev/null +++ b/DeepHermes-3-Llama-3-8B-Preview-Q4_K_S.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8f70b7b0f7e898ce80746f5cae0694190dc81a833f3e092e4b76fbe8bc788756 +size 4692675200 diff --git a/DeepHermes-3-Llama-3-8B-Preview-Q5_0.gguf b/DeepHermes-3-Llama-3-8B-Preview-Q5_0.gguf new file mode 100644 index 0000000..88b1363 --- /dev/null +++ b/DeepHermes-3-Llama-3-8B-Preview-Q5_0.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:841944a025994ad88b7a4b48866b5e87661afa572315ed3639bbaaa03f4900de +size 5599300224 diff --git a/DeepHermes-3-Llama-3-8B-Preview-Q5_K_M.gguf b/DeepHermes-3-Llama-3-8B-Preview-Q5_K_M.gguf new file mode 100644 index 0000000..811365b --- /dev/null +++ b/DeepHermes-3-Llama-3-8B-Preview-Q5_K_M.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8b56f92e50040e5b66ba7d1e20de0a8a674e27f11ce18fe0b34fb5b5e13e87ed +size 5732993664 diff --git a/DeepHermes-3-Llama-3-8B-Preview-Q5_K_S.gguf b/DeepHermes-3-Llama-3-8B-Preview-Q5_K_S.gguf new file mode 100644 index 0000000..5c70d6c --- /dev/null +++ b/DeepHermes-3-Llama-3-8B-Preview-Q5_K_S.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9a3430e8f1eeef081f3410c72045b5fed8cde15af15460c43a307c429cda011f +size 5599300224 diff --git a/DeepHermes-3-Llama-3-8B-Preview-Q6_K.gguf b/DeepHermes-3-Llama-3-8B-Preview-Q6_K.gguf new file mode 100644 index 0000000..7424f86 --- /dev/null +++ b/DeepHermes-3-Llama-3-8B-Preview-Q6_K.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2cb138f445b6c9e39bbe67d4cd82f91960d3943c411b015751b1279b13218a2d +size 6596012672 diff --git a/DeepHermes-3-Llama-3-8B-Preview-Q8_0.gguf b/DeepHermes-3-Llama-3-8B-Preview-Q8_0.gguf new file mode 100644 index 0000000..3c7b97d --- /dev/null +++ b/DeepHermes-3-Llama-3-8B-Preview-Q8_0.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3edf79cb8b4eec74a64ca7c946949bdc3f571686d3d13a287b31071b197835a6 +size 8540777088 diff --git a/DeepHermes-3-Llama-3-8B-Preview-f16.gguf b/DeepHermes-3-Llama-3-8B-Preview-f16.gguf new file mode 100644 index 0000000..dc6f7df --- /dev/null +++ b/DeepHermes-3-Llama-3-8B-Preview-f16.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:54e3e676ef0eb350b55b243b45eda41398ec59cfa41fbdae857264d2a4084d7e +size 16068897408 diff --git a/README.md b/README.md new file mode 100644 index 0000000..e85d792 --- /dev/null +++ b/README.md @@ -0,0 +1,123 @@ +--- +license: llama3 +base_model: NousResearch/DeepHermes-3-Llama-3-8B-Preview +model_creator: NousResearch +model_type: llama +quantized_by: Second State Inc. +library_name: transformers +language: +- en +--- + + + +
+ +
+
+ + +# Meta-DeepHermes-3-Llama-3-8B-Preview-GGUF + +## Original Model + +[NousResearch/DeepHermes-3-Llama-3-8B-Preview](https://huggingface.co/NousResearch/DeepHermes-3-Llama-3-8B-Preview) + +## Run with LlamaEdge + +- LlamaEdge version: [v0.16.8](https://github.com/LlamaEdge/LlamaEdge/releases/tag/0.16.8) and above + +- Prompt template + + **IMPORTANT**: To toggle REASONING ON, you must use the following system prompt: + + ```text + You are a deep thinking AI, you may use extremely long chains of thought to deeply consider the problem and deliberate with yourself via systematic reasoning processes to help come to a correct solution prior to answering. You should enclose your thoughts and internal monologue inside tags, and then provide your solution or response to the problem. + ``` + + - Prompt type for chat: `llama-3-chat` + + - Prompt string + + ```text + <|begin_of_text|><|start_header_id|>system<|end_header_id|> + + {{ system_prompt }}<|eot_id|><|start_header_id|>user<|end_header_id|> + + {{ user_message_1 }}<|eot_id|><|start_header_id|>assistant<|end_header_id|> + + {{ model_answer_1 }}<|eot_id|><|start_header_id|>user<|end_header_id|> + + {{ user_message_2 }}<|eot_id|><|start_header_id|>assistant<|end_header_id|> + ``` + + - Prompt type for tool use: `llama-3-tool` + + - Prompt string + + ```text + <|begin_of_text|><|start_header_id|>system<|end_header_id|> + + {system_message}<|eot_id|><|start_header_id|>user<|end_header_id|> + + Given the following functions, please respond with a JSON for a function call with its proper arguments that best answers the given prompt. + + Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}. Do not use variables. + + [{"type":"function","function":{"name":"get_current_weather","description":"Get the current weather in a given location","parameters":{"type":"object","properties":{"location":{"type":"string","description":"The city and state, e.g. San Francisco, CA"},"unit":{"type":"string","description":"The temperature unit to use. Infer this from the users location.","enum":["celsius","fahrenheit"]}},"required":["location","unit"]}}}] + + Question: {user_message}<|eot_id|><|start_header_id|>assistant<|end_header_id|> + ``` + +- Context size: `128000` + +- Run as LlamaEdge service + + - Chat + + ```bash + wasmedge --dir .:. --nn-preload default:GGML:AUTO:DeepHermes-3-Llama-3-8B-Preview-Q5_K_M.gguf \ + llama-api-server.wasm \ + --prompt-template llama-3-chat \ + --ctx-size 128000 \ + --model-name DeepHermes-3-Llama-3-8B-Preview + ``` + + - Tool use + + ```bash + wasmedge --dir .:. --nn-preload default:GGML:AUTO:DeepHermes-3-Llama-3-8B-Preview-Q5_K_M.gguf \ + llama-api-server.wasm \ + --prompt-template llama-3-tool \ + --ctx-size 128000 \ + --model-name DeepHermes-3-Llama-3-8B-Preview + ``` + +- Run as LlamaEdge command app + + ```bash + wasmedge --dir .:. --nn-preload default:GGML:AUTO:DeepHermes-3-Llama-3-8B-Preview-Q5_K_M.gguf \ + llama-chat.wasm \ + --prompt-template llama-3-chat \ + --ctx-size 128000 + ``` + +## Quantized GGUF Models + +| Name | Quant method | Bits | Size | Use case | +| ---- | ---- | ---- | ---- | ----- | +| [DeepHermes-3-Llama-3-8B-Preview-Q2_K.gguf](https://huggingface.co/second-state/DeepHermes-3-Llama-3-8B-Preview-GGUF/blob/main/DeepHermes-3-Llama-3-8B-Preview-Q2_K.gguf) | Q2_K | 2 | 3.18 GB| smallest, significant quality loss - not recommended for most purposes | +| [DeepHermes-3-Llama-3-8B-Preview-Q3_K_L.gguf](https://huggingface.co/second-state/DeepHermes-3-Llama-3-8B-Preview-GGUF/blob/main/DeepHermes-3-Llama-3-8B-Preview-Q3_K_L.gguf) | Q3_K_L | 3 | 4.32 GB| small, substantial quality loss | +| [DeepHermes-3-Llama-3-8B-Preview-Q3_K_M.gguf](https://huggingface.co/second-state/DeepHermes-3-Llama-3-8B-Preview-GGUF/blob/main/DeepHermes-3-Llama-3-8B-Preview-Q3_K_M.gguf) | Q3_K_M | 3 | 4.02 GB| very small, high quality loss | +| [DeepHermes-3-Llama-3-8B-Preview-Q3_K_S.gguf](https://huggingface.co/second-state/DeepHermes-3-Llama-3-8B-Preview-GGUF/blob/main/DeepHermes-3-Llama-3-8B-Preview-Q3_K_S.gguf) | Q3_K_S | 3 | 3.66 GB| very small, high quality loss | +| [DeepHermes-3-Llama-3-8B-Preview-Q4_0.gguf](https://huggingface.co/second-state/DeepHermes-3-Llama-3-8B-Preview-GGUF/blob/main/DeepHermes-3-Llama-3-8B-Preview-Q4_0.gguf) | Q4_0 | 4 | 4.66 GB| legacy; small, very high quality loss - prefer using Q3_K_M | +| [DeepHermes-3-Llama-3-8B-Preview-Q4_K_M.gguf](https://huggingface.co/second-state/DeepHermes-3-Llama-3-8B-Preview-GGUF/blob/main/DeepHermes-3-Llama-3-8B-Preview-Q4_K_M.gguf) | Q4_K_M | 4 | 4.92 GB| medium, balanced quality - recommended | +| [DeepHermes-3-Llama-3-8B-Preview-Q4_K_S.gguf](https://huggingface.co/second-state/DeepHermes-3-Llama-3-8B-Preview-GGUF/blob/main/DeepHermes-3-Llama-3-8B-Preview-Q4_K_S.gguf) | Q4_K_S | 4 | 4.69 GB| small, greater quality loss | +| [DeepHermes-3-Llama-3-8B-Preview-Q5_0.gguf](https://huggingface.co/second-state/DeepHermes-3-Llama-3-8B-Preview-GGUF/blob/main/DeepHermes-3-Llama-3-8B-Preview-Q5_0.gguf) | Q5_0 | 5 | 5.6 GB| legacy; medium, balanced quality - prefer using Q4_K_M | +| [DeepHermes-3-Llama-3-8B-Preview-Q5_K_M.gguf](https://huggingface.co/second-state/DeepHermes-3-Llama-3-8B-Preview-GGUF/blob/main/DeepHermes-3-Llama-3-8B-Preview-Q5_K_M.gguf) | Q5_K_M | 5 | 5.73 GB| large, very low quality loss - recommended | +| [DeepHermes-3-Llama-3-8B-Preview-Q5_K_S.gguf](https://huggingface.co/second-state/DeepHermes-3-Llama-3-8B-Preview-GGUF/blob/main/DeepHermes-3-Llama-3-8B-Preview-Q5_K_S.gguf) | Q5_K_S | 5 | 5.6 GB| large, low quality loss - recommended | +| [DeepHermes-3-Llama-3-8B-Preview-Q6_K.gguf](https://huggingface.co/second-state/DeepHermes-3-Llama-3-8B-Preview-GGUF/blob/main/DeepHermes-3-Llama-3-8B-Preview-Q6_K.gguf) | Q6_K | 6 | 6.6 GB| very large, extremely low quality loss | +| [DeepHermes-3-Llama-3-8B-Preview-Q8_0.gguf](https://huggingface.co/second-state/DeepHermes-3-Llama-3-8B-Preview-GGUF/blob/main/DeepHermes-3-Llama-3-8B-Preview-Q8_0.gguf) | Q8_0 | 8 | 8.54 GB| very large, extremely low quality loss - not recommended | +| [DeepHermes-3-Llama-3-8B-Preview-f16.gguf](https://huggingface.co/second-state/DeepHermes-3-Llama-3-8B-Preview-GGUF/blob/main/DeepHermes-3-Llama-3-8B-Preview-f16.gguf) | f16 | 16 | 16.1 GB| | + +*Quantized with llama.cpp b4743.* \ No newline at end of file diff --git a/config.json b/config.json new file mode 100644 index 0000000..67c0964 --- /dev/null +++ b/config.json @@ -0,0 +1,38 @@ +{ + "_name_or_path": "unsloth/Meta-Llama-3.1-8B", + "architectures": [ + "LlamaForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 128000, + "eos_token_id": 128009, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 4096, + "initializer_range": 0.02, + "intermediate_size": 14336, + "max_position_embeddings": 131072, + "mlp_bias": false, + "model_type": "llama", + "num_attention_heads": 32, + "num_hidden_layers": 32, + "num_key_value_heads": 8, + "pad_token_id": 128004, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_scaling": { + "factor": 8.0, + "high_freq_factor": 4.0, + "low_freq_factor": 1.0, + "original_max_position_embeddings": 8192, + "rope_type": "llama3" + }, + "rope_theta": 500000.0, + "tie_word_embeddings": false, + "torch_dtype": "bfloat16", + "transformers_version": "4.46.3", + "unsloth_fixed": true, + "use_cache": true, + "vocab_size": 128256 +}