From 53eeb8718e8d59a9e8e09cfb9b77a7fc06f37f98 Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Fri, 24 Apr 2026 04:51:57 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: neuralmagic/Sparse-Llama-3.1-8B-2of4 Source: Original Platform --- .gitattributes | 36 ++++++++ README.md | 148 +++++++++++++++++++++++++++++++ config.json | 3 + configuration.json | 3 + generation_config.json | 3 + model-00001-of-00004.safetensors | 3 + model-00002-of-00004.safetensors | 3 + model-00003-of-00004.safetensors | 3 + model-00004-of-00004.safetensors | 3 + model.safetensors.index.json | 3 + special_tokens_map.json | 3 + tokenizer.json | 3 + tokenizer_config.json | 3 + 13 files changed, 217 insertions(+) create mode 100644 .gitattributes create mode 100644 README.md create mode 100644 config.json create mode 100644 configuration.json create mode 100644 generation_config.json create mode 100644 model-00001-of-00004.safetensors create mode 100644 model-00002-of-00004.safetensors create mode 100644 model-00003-of-00004.safetensors create mode 100644 model-00004-of-00004.safetensors create mode 100644 model.safetensors.index.json create mode 100644 special_tokens_map.json create mode 100644 tokenizer.json create mode 100644 tokenizer_config.json diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..c4cb545 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,36 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +*.json filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..0d53180 --- /dev/null +++ b/README.md @@ -0,0 +1,148 @@ +--- +tags: +- vllm +- sparsity +pipeline_tag: text-generation +license: llama3.1 +base_model: meta-llama/Llama-3.1-8B +--- + +# Get Started +Sparse-Llama-3.1 models use 2:4 semi-structured sparsity to deliver 2x model size and compute reduction. +Explore the [launch blog](https://neuralmagic.com/blog/24-sparse-llama-smaller-models-for-efficient-gpu-inference/) to learn more about Sparse-Llama-3.1 and its potential for efficient, scalable AI deployments. +You can also find all available models in our [Neural Magic HuggingFace collection](https://huggingface.co/collections/neuralmagic/sparse-llama-31-2of4-673f6e96ae74efa213cf1cff). + +**Looking to build on top of sparse models?** Whether you aim to reduce deployment costs, improve inference performance, or create highly optimized versions for your enterprise needs, Sparse Llama provides the ideal foundation. These models offer state-of-the-art efficiency with 2:4 structured sparsity, enabling cost-effective scaling without sacrificing accuracy. +[Connect with us](https://neuralmagic.com/book-a-demo/) to explore how we can help integrate sparsity into your AI workflows. + + +# Sparse-Llama-3.1-8B-2of4 + +## Model Overview +- **Model Architecture:** Llama-3.1-8B + - **Input:** Text + - **Output:** Text +- **Model Optimizations:** + - **Sparsity:** 2:4 +- **Release Date:** 11/20/2024 +- **Version:** 1.0 +- **License(s):** [llama3.1](https://huggingface.co/meta-llama/Meta-Llama-3.1-8B/blob/main/LICENSE) +- **Model Developers:** Neural Magic + +This is the 2:4 sparse version of [Llama-3.1-8B](https://huggingface.co/meta-llama/Llama-3.1-8B). +On the [OpenLLM](https://huggingface.co/spaces/open-llm-leaderboard/open_llm_leaderboard) benchmark (version 1), it achieves an average score of 62.16, compared to 63.19 for the dense model—demonstrating a **98.37% accuracy recovery**. On the [Mosaic Eval Gauntlet](https://github.com/mosaicml/llm-foundry/blob/main/scripts/eval/local_data/EVAL_GAUNTLET.md) benchmark (version v0.3), it achieves an average score of 53.85, versus 55.34 for the dense model—representing a **97.3% accuracy recovery**. + + +### Model Optimizations + +This model was obtained by pruning all linear operators within transformer blocks to the 2:4 sparsity pattern: in each group of four weights, two are retained while two are pruned. In addition to pruning, the sparse model was trained with knowledge distillation for 13B tokens to recover the accuracy loss incurred by pruning. For pruning, we utilize optimized version of [SparseGPT](https://arxiv.org/abs/2301.00774) through [LLM-Compressor](https://github.com/vllm-project/llm-compressor), and for sparse training with knowledge distillation we utilize [SquareHead approach](https://arxiv.org/abs/2310.06927). + + +## Deployment with vLLM + +This model can be deployed efficiently using the [vLLM](https://docs.vllm.ai/en/latest/) backend. vLLM aslo supports OpenAI-compatible serving. See the [documentation](https://docs.vllm.ai/en/latest/) for more details. + + +## Evaluation + +This model was evaluated on the [OpenLLM](https://huggingface.co/spaces/open-llm-leaderboard/open_llm_leaderboard) benchmark (version 1) with the [vLLM](https://docs.vllm.ai/en/stable/) engine for faster inference. In addition to the OpenLLM benchmark, the model was evaluated on the [Mosaic Eval Gauntlet](https://github.com/mosaicml/llm-foundry/blob/main/scripts/eval/local_data/EVAL_GAUNTLET.md) benchmark (version v0.3). The evaluation results are summarized below. + + +### Accuracy + +#### Open LLM Leaderboard evaluation scores + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
BenchmarkLlama-3.1-8BSparse-Llama-3.1-8B-2of4
ARC-C (25-shot)58.259.4
MMLU (5-shot)65.460.6
HellaSwag (10-shot)82.379.8
WinoGrande (5-shot)78.375.9
GSM8K (5-shot)50.756.3
TruthfulQA (0-shot)44.240.9
Average Score63.1962.16
Accuracy Recovery (%)10098.37
+ + +#### Mosaic Eval Gauntlet evaluation scores + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
BenchmarkLlama-3.1-8BSparse-Llama-3.1-8B-2of4
World Knowledge59.455.6
Commonsense Reasoning49.350.0
Language Understanding69.869.0
Symbolic Problem Solving40.037.1
Reading Comprehension58.257.5
Average Score55.3453.85
Accuracy Recovery (%)10097.3
diff --git a/config.json b/config.json new file mode 100644 index 0000000..8756d9f --- /dev/null +++ b/config.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a911301d4d458d873929c8c64751889c3e86b203ed55369fcca5ce5284aed45 +size 892 diff --git a/configuration.json b/configuration.json new file mode 100644 index 0000000..f4dd4cf --- /dev/null +++ b/configuration.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f888421726665e8a84b738eed42a64875aed79de8be7daade851ac8bf4c0cef9 +size 73 diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..838f228 --- /dev/null +++ b/generation_config.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f0f01e92153d8d962b8abe6e5ec47c258f2660fc24fc3aa1ea6bdd4b33b2a4df +size 185 diff --git a/model-00001-of-00004.safetensors b/model-00001-of-00004.safetensors new file mode 100644 index 0000000..a23f898 --- /dev/null +++ b/model-00001-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:49d2281bc1db84a9bb86058e9c5c43bb32dd47100ead793b14ffd934b4b00bad +size 4976698672 diff --git a/model-00002-of-00004.safetensors b/model-00002-of-00004.safetensors new file mode 100644 index 0000000..ac151fb --- /dev/null +++ b/model-00002-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a5d45d17ed028bb1d3d4db6f456c4a1e036de68c54bedc5d7a462b8d23e13bfc +size 4999802720 diff --git a/model-00003-of-00004.safetensors b/model-00003-of-00004.safetensors new file mode 100644 index 0000000..df1ac50 --- /dev/null +++ b/model-00003-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a84d4a5b268728922cabb1802b08a58ac98c0fcbb43864af79f7a61edee34794 +size 4915916176 diff --git a/model-00004-of-00004.safetensors b/model-00004-of-00004.safetensors new file mode 100644 index 0000000..d190074 --- /dev/null +++ b/model-00004-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d19475b36bc9b8219311797514104976a8e580a7ee5220b38caef1721b859c29 +size 1168138808 diff --git a/model.safetensors.index.json b/model.safetensors.index.json new file mode 100644 index 0000000..a054aad --- /dev/null +++ b/model.safetensors.index.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:146776fce3f6db1103aa6f249e65ee5544c5923ce6f971b092eee79aa6e5d37b +size 23950 diff --git a/special_tokens_map.json b/special_tokens_map.json new file mode 100644 index 0000000..a8f2ae3 --- /dev/null +++ b/special_tokens_map.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc2e013b7545f183ef03e079a3c91c6f364fa37e4068c512d7dd843e59024535 +size 301 diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..66cd9d7 --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:79e3e522635f3171300913bb421464a87de6222182a0570b9b2ccba2a964b2b4 +size 9085657 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..437fd36 --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4b4f36546c6125dd40594792627b965fb5a515688a0ecdaa0afd0b5f3ffcdba3 +size 50498