commit c7dca9b71d6bdcabbfd55689b359ee2ff434319e Author: ModelHub XC Date: Thu Oct 22 16:18:12 2026 +0800 初始化项目，由ModelHub XC社区提供模型 Model: RedHatAI/Sparse-Llama-3.1-8B-evolcodealpaca-2of4-FP8-dynamic Source: Original Platform diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..52373fe --- /dev/null +++ b/.gitattributes @@ -0,0 +1,36 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..a640652 --- /dev/null +++ b/README.md @@ -0,0 +1,77 @@ +--- +tags: +- vllm +- sparsity +- quantization +- int4 +pipeline_tag: text-generation +license: llama3.1 +base_model: neuralmagic/Sparse-Llama-3.1-8B-evolcodealpaca-2of4 +datasets: +- theblackcat102/evol-codealpaca-v1 +language: +- en +--- + +# Sparse-Llama-3.1-8B-evolcodealpaca-2of4-FP8-dynamic + +## Model Overview +- **Model Architecture:** Llama-3.1-8B + - **Input:** Text + - **Output:** Text +- **Model Optimizations:** + - **Sparsity:** 2:4 + - **Weight quantization:** FP8 + - **Activation quantization:** FP8 +- **Release Date:** 11/15/2024 +- **Version:** 1.0 +- **License(s):** [llama3.1](https://huggingface.co/meta-llama/Meta-Llama-3.1-8B/blob/main/LICENSE) +- **Model Developers:** Neural Magic + +This is a code completion AI model obtained by fine-tuning the 2:4 sparse [Sparse-Llama-3.1-8B-2of4](https://huggingface.co/neuralmagic/Sparse-Llama-3.1-8B-2of4) on the [evol-codealpaca-v1](https://huggingface.co/datasets/theblackcat102/evol-codealpaca-v1) dataset, followed by quantization +On the [HumanEval](https://arxiv.org/abs/2107.03374) benchmark, it achieves a pass@1 of 49.0, compared to 48.5 for the fine-tuned dense model [Llama-3.1-8B-evolcodealpaca](https://huggingface.co/neuralmagic/Llama-3.1-8B-evolcodealpaca) — demonstrating over **100% accuracy recovery**. + + +### Model Optimizations + +This model was obtained by quantizing the weights and of [Sparse-Llama-3.1-8B-evolcodealpaca-2of4](https://huggingface.co/neuralmagic/Sparse-Llama-3.1-8B-evolcodealpaca-2of4) to FP8 data type. +This optimization reduces the number of bits used to represent weights and activations from 16 to 8, reducing GPU memory requirements (by approximately 50%) and increasing matrix-multiply compute throughput (by approximately 2x). +Weight quantization also reduces disk size requirements by approximately 50%. + +Only weights and activations of the linear operators within transformers blocks are quantized. +Weights are quantized with a symmetric static per-channel scheme, where a fixed linear scaling factor is applied between FP8 and BF16 representations for each output channel dimension. +Linear scaling factors are computed via by minimizing the mean squarred error (MSE). +Activations are quantized with a symmetric dynamic per-token scheme, computing a linear scaling factor at runtime for each token between FP8 and BF16 representations. + + +## Deployment with vLLM + +This model can be deployed efficiently using the [vLLM](https://docs.vllm.ai/en/latest/) backend. vLLM also supports OpenAI-compatible serving. See the [documentation](https://docs.vllm.ai/en/latest/) for more details. + + +## Evaluation + +This model was evaluated on Neural Magic's fork of [EvalPlus](https://github.com/neuralmagic/evalplus). + +### Accuracy +#### Human Benchmark + + + + + + + + + + + + + + + + + + + +
MetricLlama-3.1-8B-evolcodealpacaSparse-Llama-3.1-8B-evolcodealpaca-2of4Sparse-Llama-3.1-8B-evolcodealpaca-2of4-FP8-dynamic
HumanEval pass@148.549.149.0
HumanEval+ pass@144.246.346.2
\ No newline at end of file diff --git a/config.json b/config.json new file mode 100644 index 0000000..be8aa77 --- /dev/null +++ b/config.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8bf5f373bc28a87465ac9a52129433f4a98264778fab77785f16b982c893e2c5 +size 2031 diff --git a/configuration.json b/configuration.json new file mode 100644 index 0000000..bbeeda1 --- /dev/null +++ b/configuration.json @@ -0,0 +1 @@ +{"framework": "pytorch", "task": "text-generation", "allow_remote": true} \ No newline at end of file diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..388971c --- /dev/null +++ b/generation_config.json @@ -0,0 +1,9 @@ +{ + "_from_model_config": true, + "bos_token_id": 128000, + "do_sample": true, + "eos_token_id": 128001, + "temperature": 0.6, + "top_p": 0.9, + "transformers_version": "4.46.2" +} diff --git a/model-00001-of-00002.safetensors b/model-00001-of-00002.safetensors new file mode 100644 index 0000000..e72a870 --- /dev/null +++ b/model-00001-of-00002.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f74a09c25d9feeb02d65de3892b346efec4ec06f5e34cd38623b5ced57913fe8 +size 4999401504 diff --git a/model-00002-of-00002.safetensors b/model-00002-of-00002.safetensors new file mode 100644 index 0000000..ea3b968 --- /dev/null +++ b/model-00002-of-00002.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3b0a091be180134bb7726644fbafa5528b406ecd77ffeefdaaeaaedbceaae658 +size 4084612976 diff --git a/model.safetensors.index.json b/model.safetensors.index.json new file mode 100644 index 0000000..889bb55 --- /dev/null +++ b/model.safetensors.index.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7ed839beb27a9a1c3bef4e5f5b5011ba4c7dd8595770d17b85ea56df6a69d83e +size 43463 diff --git a/recipe.yaml b/recipe.yaml new file mode 100644 index 0000000..044a5bf --- /dev/null +++ b/recipe.yaml @@ -0,0 +1,10 @@ +quant_stage: + quant_modifiers: + QuantizationModifier: + ignore: [lm_head] + config_groups: + group_0: + targets: [Linear] + weights: {num_bits: 8, type: float, symmetric: true, strategy: channel, observer: mse} + input_activations: {num_bits: 8, type: float, symmetric: true, strategy: token, + dynamic: true, observer: memoryless} diff --git a/special_tokens_map.json b/special_tokens_map.json new file mode 100644 index 0000000..e5b39b6 --- /dev/null +++ b/special_tokens_map.json @@ -0,0 +1,23 @@ +{ + "bos_token": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } +} diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..1c1d8d5 --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b +size 17209920 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..678a119 --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:98a9f08f8544d9cf35ec344b5b92a0b87ad8ec7e1142510884a856371c3a74c3 +size 50532