From 47924db71d5e35cc610018a7d03e38b3fb9253d3 Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Fri, 23 Oct 2026 21:52:14 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: RedHatAI/Llama-3.2-1B-Instruct-FP8-dynamic Source: Original Platform --- .gitattributes | 36 +++++ README.md | 319 ++++++++++++++++++++++++++++++++++++++++ config.json | 3 + configuration.json | 1 + generation_config.json | 12 ++ model.safetensors | 3 + recipe.yaml | 9 ++ special_tokens_map.json | 16 ++ tokenizer.json | 3 + tokenizer_config.json | 3 + 10 files changed, 405 insertions(+) create mode 100644 .gitattributes create mode 100644 README.md create mode 100644 config.json create mode 100644 configuration.json create mode 100644 generation_config.json create mode 100644 model.safetensors create mode 100644 recipe.yaml create mode 100644 special_tokens_map.json create mode 100644 tokenizer.json create mode 100644 tokenizer_config.json diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..c28a04d --- /dev/null +++ b/.gitattributes @@ -0,0 +1,36 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..0f5ec1f --- /dev/null +++ b/README.md @@ -0,0 +1,319 @@ +--- +tags: +- fp8 +- vllm +language: +- en +- de +- fr +- it +- pt +- hi +- es +- th +pipeline_tag: text-generation +license: llama3.2 +base_model: meta-llama/Llama-3.2-1B-Instruct +--- + +# Llama-3.2-1B-Instruct-FP8-dynamic + +## Model Overview +- **Model Architecture:** Meta-Llama-3.2 + - **Input:** Text + - **Output:** Text +- **Model Optimizations:** + - **Weight quantization:** FP8 + - **Activation quantization:** FP8 +- **Intended Use Cases:** Intended for commercial and research use in multiple languages. Similarly to [Llama-3.2-1B-Instruct](https://huggingface.co/meta-llama/Llama-3.2-1B-Instruct), this models is intended for assistant-like chat. +- **Out-of-scope:** Use in any manner that violates applicable laws or regulations (including trade compliance laws). Use in languages other than English. +- **Release Date:** 9/25/2024 +- **Version:** 1.0 +- **License(s):** [llama3.2](https://huggingface.co/meta-llama/Llama-3.2-1B/blob/main/LICENSE) +- **Model Developers:** Neural Magic + +Quantized version of [Llama-3.2-1B-Instruct](https://huggingface.co/meta-llama/Llama-3.2-1B-Instruct). +It achieves an average score of 50.88 on a subset of task from the [OpenLLM](https://huggingface.co/spaces/open-llm-leaderboard/open_llm_leaderboard) benchmark (version 1), whereas the unquantized model achieves 51.70. + +### Model Optimizations + +This model was obtained by quantizing the weights and activations of [Llama-3.2-1B-Instruct](https://huggingface.co/meta-llama/Llama-3.2-1B-Instruct) to FP8 data type, ready for inference with vLLM built from source. +This optimization reduces the number of bits per parameter from 16 to 8, reducing the disk size and GPU memory requirements by approximately 50%. + +Only the weights and activations of the linear operators within transformers blocks are quantized. Symmetric per-channel quantization is applied, in which a linear scaling per output dimension maps the FP8 representations of the quantized weights and activations. Activations are also quantized on a per-token dynamic basis. +[LLM Compressor](https://github.com/vllm-project/llm-compressor) is used for quantization. + +## Deployment + +### Use with vLLM + +This model can be deployed efficiently using the [vLLM](https://docs.vllm.ai/en/latest/) backend, as shown in the example below. + +```python +from vllm import LLM, SamplingParams +from transformers import AutoTokenizer + +model_id = "neuralmagic/Llama-3.2-1B-Instruct-FP8-dynamic" + +sampling_params = SamplingParams(temperature=0.6, top_p=0.9, max_tokens=256) + +tokenizer = AutoTokenizer.from_pretrained(model_id) + +messages = [ + {"role": "system", "content": "You are a pirate chatbot who always responds in pirate speak!"}, + {"role": "user", "content": "Who are you?"}, +] + +prompts = tokenizer.apply_chat_template(messages, tokenize=False) + +llm = LLM(model=model_id) + +outputs = llm.generate(prompts, sampling_params) + +generated_text = outputs[0].outputs[0].text +print(generated_text) +``` + +vLLM aslo supports OpenAI-compatible serving. See the [documentation](https://docs.vllm.ai/en/latest/) for more details. + +## Creation + +This model was created by applying [LLM Compressor](https://github.com/vllm-project/llm-compressor/blob/sa/big_model_support/examples/big_model_offloading/big_model_w8a8_calibrate.py), as presented in the code snipet below. + +```python +import torch + +from transformers import AutoTokenizer + +from llmcompressor.transformers import SparseAutoModelForCausalLM, oneshot +from llmcompressor.transformers.compression.helpers import ( # noqa + calculate_offload_device_map, + custom_offload_device_map, +) + +recipe = """ +quant_stage: + quant_modifiers: + QuantizationModifier: + ignore: ["lm_head"] + config_groups: + group_0: + weights: + num_bits: 8 + type: float + strategy: channel + dynamic: false + symmetric: true + input_activations: + num_bits: 8 + type: float + strategy: token + dynamic: true + symmetric: true + targets: ["Linear"] +""" + +model_stub = "meta-llama/Llama-3.2-1B-Instruct" +model_name = model_stub.split("/")[-1] + +device_map = calculate_offload_device_map( + model_stub, reserve_for_hessians=False, num_gpus=1, torch_dtype="auto" +) + +model = SparseAutoModelForCausalLM.from_pretrained( + model_stub, torch_dtype="auto", device_map=device_map +) + +output_dir = f"./{model_name}-FP8-dynamic" + +oneshot( + model=model, + recipe=recipe, + output_dir=output_dir, + save_compressed=True, + tokenizer=AutoTokenizer.from_pretrained(model_stub), +) +``` + +## Evaluation + +The model was evaluated on MMLU, ARC-Challenge, GSM-8K, and Winogrande. +Evaluation was conducted using the Neural Magic fork of [lm-evaluation-harness](https://github.com/neuralmagic/lm-evaluation-harness/tree/llama_3.1_instruct) (branch llama_3.1_instruct) and the [vLLM](https://docs.vllm.ai/en/stable/) engine. +This version of the lm-evaluation-harness includes versions of ARC-Challenge, GSM-8K, MMLU, and MMLU-cot that match the prompting style of [Meta-Llama-3.1-Instruct-evals](https://huggingface.co/datasets/meta-llama/Meta-Llama-3.1-8B-Instruct-evals). + +### Accuracy + +#### Open LLM Leaderboard evaluation scores + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Benchmark + Llama-3.2-1B-Instruct + Llama-3.2-1B-Instruct-FP8-dynamic (this model) + Recovery +
MMLU (5-shot) + 47.66 + 47.55 + 99.8% +
MMLU-cot (0-shot) + 47.10 + 46.79 + 99.3% +
ARC Challenge (0-shot) + 58.36 + 57.25 + 98.1% +
GSM-8K-cot (8-shot, strict-match) + 45.72 + 45.94 + 100.5% +
Winogrande (5-shot) + 62.27 + 61.40 + 98.6% +
Hellaswag (10-shot) + 61.01 + 60.95 + 99.9% +
TruthfulQA (0-shot, mc2) + 43.52 + 44.23 + 101.6% +
Average + 52.24 + 52.02 + 99.7% +
+ +### Reproduction + +The results were obtained using the following commands: + + +#### MMLU +``` +lm_eval \ + --model vllm \ + --model_args pretrained="neuralmagic/Llama-3.2-1B-Instruct-FP8-dynamic",dtype=auto,max_model_len=3850,max_gen_toks=10,tensor_parallel_size=1 \ + --tasks mmlu_llama_3.1_instruct \ + --fewshot_as_multiturn \ + --apply_chat_template \ + --num_fewshot 5 \ + --batch_size auto +``` + +#### MMLU-CoT +``` +lm_eval \ + --model vllm \ + --model_args pretrained="neuralmagic/Llama-3.2-1B-Instruct-FP8-dynamic",dtype=auto,max_model_len=4064,max_gen_toks=1024,tensor_parallel_size=1 \ + --tasks mmlu_cot_0shot_llama_3.1_instruct \ + --apply_chat_template \ + --num_fewshot 0 \ + --batch_size auto +``` + +#### ARC-Challenge +``` +lm_eval \ + --model vllm \ + --model_args pretrained="neuralmagic/Llama-3.2-1B-Instruct-FP8-dynamic",dtype=auto,max_model_len=3940,max_gen_toks=100,tensor_parallel_size=1 \ + --tasks arc_challenge_llama_3.1_instruct \ + --apply_chat_template \ + --num_fewshot 0 \ + --batch_size auto +``` + +#### GSM-8K +``` +lm_eval \ + --model vllm \ + --model_args pretrained="neuralmagic/Llama-3.2-1B-Instruct-FP8-dynamic",dtype=auto,max_model_len=4096,max_gen_toks=1024,tensor_parallel_size=1 \ + --tasks gsm8k_cot_llama_3.1_instruct \ + --fewshot_as_multiturn \ + --apply_chat_template \ + --num_fewshot 8 \ + --batch_size auto +``` + +#### Hellaswag +``` +lm_eval \ + --model vllm \ + --model_args pretrained="neuralmagic/Llama-3.2-1B-Instruct-FP8-dynamic",dtype=auto,add_bos_token=True,max_model_len=4096,tensor_parallel_size=1 \ + --tasks hellaswag \ + --num_fewshot 10 \ + --batch_size auto +``` + +#### Winogrande +``` +lm_eval \ + --model vllm \ + --model_args pretrained="neuralmagic/Llama-3.2-1B-Instruct-FP8-dynamic",dtype=auto,add_bos_token=True,max_model_len=4096,tensor_parallel_size=1 \ + --tasks winogrande \ + --num_fewshot 5 \ + --batch_size auto +``` + +#### TruthfulQA +``` +lm_eval \ + --model vllm \ + --model_args pretrained="neuralmagic/Llama-3.2-1B-Instruct-FP8-dynamic",dtype=auto,add_bos_token=True,max_model_len=4096,tensor_parallel_size=1 \ + --tasks truthfulqa \ + --num_fewshot 0 \ + --batch_size auto +``` \ No newline at end of file diff --git a/config.json b/config.json new file mode 100644 index 0000000..ca32491 --- /dev/null +++ b/config.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:930f48644b5d4b522a29bf4e2e500e660aef9fa0cd55a7391d08b041be6eedf9 +size 2104 diff --git a/configuration.json b/configuration.json new file mode 100644 index 0000000..bbeeda1 --- /dev/null +++ b/configuration.json @@ -0,0 +1 @@ +{"framework": "pytorch", "task": "text-generation", "allow_remote": true} \ No newline at end of file diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..6a807a3 --- /dev/null +++ b/generation_config.json @@ -0,0 +1,12 @@ +{ + "bos_token_id": 128000, + "do_sample": true, + "eos_token_id": [ + 128001, + 128008, + 128009 + ], + "temperature": 0.6, + "top_p": 0.9, + "transformers_version": "4.45.0" +} diff --git a/model.safetensors b/model.safetensors new file mode 100644 index 0000000..dfd1478 --- /dev/null +++ b/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e24862bf8898e483b30c924721debd7b622ef69e0f78d6552c3bd605b5d7cd19 +size 2024671096 diff --git a/recipe.yaml b/recipe.yaml new file mode 100644 index 0000000..9c7755f --- /dev/null +++ b/recipe.yaml @@ -0,0 +1,9 @@ +quant_stage: + quant_modifiers: + QuantizationModifier: + ignore: [lm_head] + config_groups: + group_0: + weights: {num_bits: 8, type: float, strategy: channel, dynamic: false, symmetric: true} + input_activations: {num_bits: 8, type: float, strategy: token, dynamic: true, symmetric: true} + targets: [Linear] diff --git a/special_tokens_map.json b/special_tokens_map.json new file mode 100644 index 0000000..02ee80b --- /dev/null +++ b/special_tokens_map.json @@ -0,0 +1,16 @@ +{ + "bos_token": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "<|eot_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } +} diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..1c1d8d5 --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b +size 17209920 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..108c212 --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9823dcfdc1121869029da45192238e85cf44f0b232a6d9dc20e4fe6f4242a14e +size 54528