commit 3aff0f7c6ac0eea753d749e13aa6b00ed0d13ac9 Author: ModelHub XC Date: Wed Jun 17 23:52:12 2026 +0800 初始化项目，由ModelHub XC社区提供模型 Model: second-state/Llama-2-7B-Chat-GGUF Source: Original Platform diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..f15b49c --- /dev/null +++ b/.gitattributes @@ -0,0 +1,36 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +*.gguf filter=lfs diff=lfs merge=lfs -text diff --git a/Llama-2-7b-chat-hf-Q2_K.gguf b/Llama-2-7b-chat-hf-Q2_K.gguf new file mode 100644 index 0000000..15b81e7 --- /dev/null +++ b/Llama-2-7b-chat-hf-Q2_K.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:705421d1cb35629acb40d7f214cffdba5885ce3166c0df4c2663c7fbbebdbb4b +size 2825941600 diff --git a/Llama-2-7b-chat-hf-Q3_K_L.gguf b/Llama-2-7b-chat-hf-Q3_K_L.gguf new file mode 100644 index 0000000..2cd65b4 --- /dev/null +++ b/Llama-2-7b-chat-hf-Q3_K_L.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4f0aaff8e9bddcc0bb7222eaeed960ec897d565a65eb7f1b6b1cd479da0ab97f +size 3597111904 diff --git a/Llama-2-7b-chat-hf-Q3_K_M.gguf b/Llama-2-7b-chat-hf-Q3_K_M.gguf new file mode 100644 index 0000000..37818dd --- /dev/null +++ b/Llama-2-7b-chat-hf-Q3_K_M.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3b8c03671a35e22a7bc118f5fbe0502226d96c60c86dfa0b0ebccbb211fe8b51 +size 3298005600 diff --git a/Llama-2-7b-chat-hf-Q3_K_S.gguf b/Llama-2-7b-chat-hf-Q3_K_S.gguf new file mode 100644 index 0000000..2e81c23 --- /dev/null +++ b/Llama-2-7b-chat-hf-Q3_K_S.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ee1dd1e86097d8074cc4cf7b36596daca1dd4df312e62bb468ef27cde2e14e16 +size 2948305504 diff --git a/Llama-2-7b-chat-hf-Q4_0.gguf b/Llama-2-7b-chat-hf-Q4_0.gguf new file mode 100644 index 0000000..4751c79 --- /dev/null +++ b/Llama-2-7b-chat-hf-Q4_0.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:23a66af3672c603077b86510bee04cac74fa4da675b4318fc649624fce8c13b3 +size 3825807968 diff --git a/Llama-2-7b-chat-hf-Q4_K_M.gguf b/Llama-2-7b-chat-hf-Q4_K_M.gguf new file mode 100644 index 0000000..4b7d38f --- /dev/null +++ b/Llama-2-7b-chat-hf-Q4_K_M.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f475bcb77959f5721548bcc5122f4d5a0eb9d94ddb5b7fdce826bc1410ae0a8b +size 4081005152 diff --git a/Llama-2-7b-chat-hf-Q4_K_S.gguf b/Llama-2-7b-chat-hf-Q4_K_S.gguf new file mode 100644 index 0000000..54a504d --- /dev/null +++ b/Llama-2-7b-chat-hf-Q4_K_S.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d51f515b9a8da317c94163e26596baeb57da3a4cc0e6f422c75bfb9ea126a041 +size 3856740960 diff --git a/Llama-2-7b-chat-hf-Q5_0.gguf b/Llama-2-7b-chat-hf-Q5_0.gguf new file mode 100644 index 0000000..985ef62 --- /dev/null +++ b/Llama-2-7b-chat-hf-Q5_0.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:459aedfd356c4842be1b1571b920779c9fbd8e8f7c7d4c9218305ffaa4f29b3e +size 4651692640 diff --git a/Llama-2-7b-chat-hf-Q5_K_M.gguf b/Llama-2-7b-chat-hf-Q5_K_M.gguf new file mode 100644 index 0000000..13bc676 --- /dev/null +++ b/Llama-2-7b-chat-hf-Q5_K_M.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:39fdaca41ef03de1e9b709602557faaf2e8490c830622823cb6f8dc9ac14db04 +size 4783157856 diff --git a/Llama-2-7b-chat-hf-Q5_K_S.gguf b/Llama-2-7b-chat-hf-Q5_K_S.gguf new file mode 100644 index 0000000..580f427 --- /dev/null +++ b/Llama-2-7b-chat-hf-Q5_K_S.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:77cca6e9ae183339c9d1b23daca3d8678f303ea1da78cac47fc06cf3bc03cc5d +size 4651692640 diff --git a/Llama-2-7b-chat-hf-Q6_K.gguf b/Llama-2-7b-chat-hf-Q6_K.gguf new file mode 100644 index 0000000..635b1c4 --- /dev/null +++ b/Llama-2-7b-chat-hf-Q6_K.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4f226786216f352b8f1bac735fec0483b1ddac06c7fb9033ed9b48f9eccf89a4 +size 5529195104 diff --git a/Llama-2-7b-chat-hf-Q8_0.gguf b/Llama-2-7b-chat-hf-Q8_0.gguf new file mode 100644 index 0000000..48e1c0b --- /dev/null +++ b/Llama-2-7b-chat-hf-Q8_0.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:edc144a62dbf51a0d7e806b5bea17f621af2a508dfe84fb115c7dbf0c31e2838 +size 7161090656 diff --git a/Llama-2-7b-chat-hf-f16.gguf b/Llama-2-7b-chat-hf-f16.gguf new file mode 100644 index 0000000..a5bf00a --- /dev/null +++ b/Llama-2-7b-chat-hf-f16.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:23007e0c21faf4845ef8a04610c4f798d0314f9d3765fe36b1710e0294309af2 +size 13478105696 diff --git a/README.md b/README.md new file mode 100644 index 0000000..7d42729 --- /dev/null +++ b/README.md @@ -0,0 +1,90 @@ +--- +language: +- en +license: llama2 +tags: +- facebook +- meta +- pytorch +- llama +- llama-2 +model_name: Llama 2 7B Chat +arxiv: 2307.09288 +base_model: meta-llama/Llama-2-7b-chat-hf +inference: false +model_creator: Meta Llama 2 +model_type: llama +pipeline_tag: text-generation +quantized_by: Second State Inc. +--- + + + +
+ +
+
+ + +# Llama-2-7B-Chat-GGUF + +## Original Model + +[meta-llama/Llama-2-7b-chat-hf](https://huggingface.co/meta-llama/Llama-2-7b-chat-hf) + +## Run with LlamaEdge + +- LlamaEdge version: [v0.2.8](https://github.com/LlamaEdge/LlamaEdge/releases/tag/0.2.8) and above + +- Prompt template + + - Prompt type: `llama-2-chat` + + - Prompt string + + ```text + [INST] <> + {{ system_prompt }} + <> + + {{ user_msg_1 }} [/INST] {{ model_answer_1 }} [INST] {{ user_msg_2 }} [/INST] + ``` + +- Context size: `4096` + +- Run as LlamaEdge service + + ```bash + wasmedge --dir .:. --nn-preload default:GGML:AUTO:Llama-2-7b-chat-hf-Q5_K_M.gguf \ + llama-api-server.wasm \ + --prompt-template llama-2-chat \ + --ctx-size 4096 \ + --model-name llama-2-7b-chat + ``` + +- Run as LlamaEdge command app + + ```bash + wasmedge --dir .:. --nn-preload default:GGML:AUTO:Llama-2-7b-chat-hf-Q5_K_M.gguf \ + llama-chat.wasm \ + --prompt-template llama-2-chat \ + --ctx-size 4096 + ``` + +## Quantized GGUF Models + +| Name | Quant method | Bits | Size | Use case | +| ---- | ---- | ---- | ---- | ----- | +| [Llama-2-7b-chat-hf-Q2_K.gguf](https://huggingface.co/second-state/Llama-2-7B-Chat-GGUF/blob/main/Llama-2-7b-chat-hf-Q2_K.gguf) | Q2_K | 2 | 2.83 GB| smallest, significant quality loss - not recommended for most purposes | +| [Llama-2-7b-chat-hf-Q3_K_L.gguf](https://huggingface.co/second-state/Llama-2-7B-Chat-GGUF/blob/main/Llama-2-7b-chat-hf-Q3_K_L.gguf) | Q3_K_L | 3 | 3.6 GB| small, substantial quality loss | +| [Llama-2-7b-chat-hf-Q3_K_M.gguf](https://huggingface.co/second-state/Llama-2-7B-Chat-GGUF/blob/main/Llama-2-7b-chat-hf-Q3_K_M.gguf) | Q3_K_M | 3 | 3.3 GB| very small, high quality loss | +| [Llama-2-7b-chat-hf-Q3_K_S.gguf](https://huggingface.co/second-state/Llama-2-7B-Chat-GGUF/blob/main/Llama-2-7b-chat-hf-Q3_K_S.gguf) | Q3_K_S | 3 | 2.95 GB| very small, high quality loss | +| [Llama-2-7b-chat-hf-Q4_0.gguf](https://huggingface.co/second-state/Llama-2-7B-Chat-GGUF/blob/main/Llama-2-7b-chat-hf-Q4_0.gguf) | Q4_0 | 4 | 3.83 GB| legacy; small, very high quality loss - prefer using Q3_K_M | +| [Llama-2-7b-chat-hf-Q4_K_M.gguf](https://huggingface.co/second-state/Llama-2-7B-Chat-GGUF/blob/main/Llama-2-7b-chat-hf-Q4_K_M.gguf) | Q4_K_M | 4 | 4.08 GB| medium, balanced quality - recommended | +| [Llama-2-7b-chat-hf-Q4_K_S.gguf](https://huggingface.co/second-state/Llama-2-7B-Chat-GGUF/blob/main/Llama-2-7b-chat-hf-Q4_K_S.gguf) | Q4_K_S | 4 | 3.86 GB| small, greater quality loss | +| [Llama-2-7b-chat-hf-Q5_0.gguf](https://huggingface.co/second-state/Llama-2-7B-Chat-GGUF/blob/main/Llama-2-7b-chat-hf-Q5_0.gguf) | Q5_0 | 5 | 4.65 GB| legacy; medium, balanced quality - prefer using Q4_K_M | +| [Llama-2-7b-chat-hf-Q5_K_M.gguf](https://huggingface.co/second-state/Llama-2-7B-Chat-GGUF/blob/main/Llama-2-7b-chat-hf-Q5_K_M.gguf) | Q5_K_M | 5 | 4.78 GB| large, very low quality loss - recommended | +| [Llama-2-7b-chat-hf-Q5_K_S.gguf](https://huggingface.co/second-state/Llama-2-7B-Chat-GGUF/blob/main/Llama-2-7b-chat-hf-Q5_K_S.gguf) | Q5_K_S | 5 | 4.65 GB| large, low quality loss - recommended | +| [Llama-2-7b-chat-hf-Q6_K.gguf](https://huggingface.co/second-state/Llama-2-7B-Chat-GGUF/blob/main/Llama-2-7b-chat-hf-Q6_K.gguf) | Q6_K | 6 | 5.53 GB| very large, extremely low quality loss | +| [Llama-2-7b-chat-hf-Q8_0.gguf](https://huggingface.co/second-state/Llama-2-7B-Chat-GGUF/blob/main/Llama-2-7b-chat-hf-Q8_0.gguf) | Q8_0 | 8 | 7.16 GB| very large, extremely low quality loss - not recommended | +| [Llama-2-7b-chat-hf-f16.gguf](https://huggingface.co/second-state/Llama-2-7B-Chat-GGUF/blob/main/Llama-2-7b-chat-hf-f16.gguf) | f16 | 16 | 13.5 GB| | diff --git a/config.json b/config.json new file mode 100644 index 0000000..a6dccd7 --- /dev/null +++ b/config.json @@ -0,0 +1 @@ +Repo model meta-llama/Llama-2-13b-chat-hf is gated. You must be authenticated to access it. \ No newline at end of file diff --git a/configuration.json b/configuration.json new file mode 100644 index 0000000..bbeeda1 --- /dev/null +++ b/configuration.json @@ -0,0 +1 @@ +{"framework": "pytorch", "task": "text-generation", "allow_remote": true} \ No newline at end of file diff --git a/llama-2-7b-chat.Q5_K_M.gguf b/llama-2-7b-chat.Q5_K_M.gguf new file mode 100644 index 0000000..4878b3e --- /dev/null +++ b/llama-2-7b-chat.Q5_K_M.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e0b99920cf47b94c78d2fb06a1eceb9ed795176dfa3f7feac64629f1b52b997f +size 4783156928