commit 9bd5c221a71dfec5e98d862c7df3a03a2798e446 Author: ModelHub XC Date: Sun Sep 20 08:38:12 2026 +0800 初始化项目，由ModelHub XC社区提供模型 Model: RedHatAI/pixtral-12b-FP8-dynamic Source: Original Platform diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..52373fe --- /dev/null +++ b/.gitattributes @@ -0,0 +1,36 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..5ce099e --- /dev/null +++ b/README.md @@ -0,0 +1,151 @@ +--- +tags: +- fp8 +- vllm +language: +- en +- de +- fr +- it +- pt +- hi +- es +- th +pipeline_tag: text-generation +license: apache-2.0 +library_name: vllm +base_model: +- mistral-community/pixtral-12b +- mistralai/Pixtral-12B-2409 +--- + +# pixtral-12b-FP8-dynamic + +## Model Overview +- **Model Architecture:** Pixtral (Llava) + - **Input:** Text/Image + - **Output:** Text +- **Model Optimizations:** + - **Weight quantization:** FP8 + - **Activation quantization:** FP8 +- **Intended Use Cases:** Intended for commercial and research use in multiple languages. Similar to [mistralai/Pixtral-12B-2409](https://huggingface.co/mistralai/Pixtral-12B-2409), this models is intended for assistant-like chat. +- **Out-of-scope:** Use in any manner that violates applicable laws or regulations (including trade compliance laws). Use in languages other than English. +- **Release Date:** 11/1/2024 +- **Version:** 1.0 +- **License(s):** [Apache 2.0](https://huggingface.co/datasets/choosealicense/licenses/blob/main/markdown/apache-2.0.md) +- **Model Developers:** Neural Magic + +Quantized version of [mistral-community/pixtral-12b](https://huggingface.co/mistral-community/pixtral-12b). + +### Model Optimizations + +This model was obtained by quantizing the weights and activations of [mistral-community/pixtral-12b](https://huggingface.co/mistral-community/pixtral-12b) to FP8 data type, ready for inference with vLLM built from source. +This optimization reduces the number of bits per parameter from 16 to 8, reducing the disk size and GPU memory requirements by approximately 50%. + +Only the weights and activations of the linear operators within transformers blocks are quantized. Symmetric per-channel quantization is applied, in which a linear scaling per output dimension maps the FP8 representations of the quantized weights and activations. Activations are also quantized on a per-token dynamic basis. +[LLM Compressor](https://github.com/vllm-project/llm-compressor) is used for quantization. + +## Deployment + +### Use with vLLM + +This model can be deployed efficiently using the [vLLM](https://docs.vllm.ai/en/latest/) backend, as shown in the example below. + +```python +from vllm import LLM, SamplingParams + +# Initialize the LLM +model_name = "neuralmagic/pixtral-12b-FP8-dynamic" +llm = LLM(model=model_name, max_model_len=10000) + +# Create the prompt +image_url = "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg" +messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Describe the image."}, + {"type": "image_url", "image_url": {"url": image_url}}, + ], + }, +] + +# Set up sampling parameters +sampling_params = SamplingParams(temperature=0.2, max_tokens=100) + +# Generate the response +outputs = llm.chat(messages, sampling_params=sampling_params) + +# Print the generated text +for output in outputs: + print(output.outputs[0].text) +``` + +vLLM also supports OpenAI-compatible serving. See the [documentation](https://docs.vllm.ai/en/latest/) for more details. + +``` +vllm serve neuralmagic/pixtral-12b-FP8-dynamic +``` + +## Creation + +This model was created by applying [LLM Compressor](https://github.com/vllm-project/llm-compressor/blob/f90013702b15bd1690e4e2fe9ed434921b6a6199/examples/quantization_w8a8_fp8/llama3.2_vision_example.py), as presented in the code snipet below. + +```python +from transformers import AutoProcessor, LlavaForConditionalGeneration + +from llmcompressor.modifiers.quantization import QuantizationModifier +from llmcompressor.transformers import oneshot, wrap_hf_model_class + +MODEL_ID = "mistral-community/pixtral-12b" + +# Load model. +model_class = wrap_hf_model_class(LlavaForConditionalGeneration) +model = model_class.from_pretrained(MODEL_ID, device_map="auto", torch_dtype="auto") +processor = AutoProcessor.from_pretrained(MODEL_ID) + +# Configure the quantization algorithm and scheme. +# In this case, we: +# * quantize the weights to fp8 with per channel via ptq +# * quantize the activations to fp8 with dynamic per token +recipe = QuantizationModifier( + targets="Linear", + scheme="FP8_DYNAMIC", + ignore=["re:.*lm_head", "re:multi_modal_projector.*", "re:vision_model.*"], +) + +# Apply quantization and save to disk in compressed-tensors format. +SAVE_DIR = MODEL_ID.split("/")[1] + "-FP8-Dynamic" +oneshot(model=model, recipe=recipe, output_dir=SAVE_DIR) +processor.save_pretrained(SAVE_DIR) + +# Confirm generations of the quantized model look sane. +print("========== SAMPLE GENERATION ==============") +input_ids = processor(text="Hello my name is", return_tensors="pt").input_ids.to("cuda") +output = model.generate(input_ids, max_new_tokens=20) +print(processor.decode(output[0])) +print("==========================================") +``` + +## Evaluation + +### Multimodal Benchmarks + +| | pixtral-12b | pixtral-12b-FP8-dynamic | +|:-------------------:|:-------------:|:----------:| +| **MMMU** *(CoT)* | 49.44 | 51.11 | +| **Mathvista** *(CoT)* | 58.1 | 59.4 | +| **ChartQA** *(CoT)* | 82.64 | 82.68 | +| **DocVQA** *(ANLS)* | 89.36 | 89.35 | + +### Text Benchmarks + +| | pixtral-12b | pixtral-12b-FP8-dynamic | +|:-------------------:|:-------------:|:----------:| +| **MMLU** *(5-shot)* | 69.27 | 68.96 | +| **Math** *(0-shot)* | 43.82 | 43.27 | +| **Human Eval** *(Pass@1)* | 77.80 | 76.4 | + +### Reproduction + +TBD diff --git a/chat_template.json b/chat_template.json new file mode 100644 index 0000000..3ede031 --- /dev/null +++ b/chat_template.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5fe90d201d7743edfee00755a19a13d7cd6c0e09df81e1520358bbfd113d7141 +size 1628 diff --git a/config.json b/config.json new file mode 100644 index 0000000..7f288bb --- /dev/null +++ b/config.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:858374bfc356827387dd501a5683b990f56ff39f7d7d18217da852624fd42608 +size 16373 diff --git a/configuration.json b/configuration.json new file mode 100644 index 0000000..bbeeda1 --- /dev/null +++ b/configuration.json @@ -0,0 +1 @@ +{"framework": "pytorch", "task": "text-generation", "allow_remote": true} \ No newline at end of file diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..31dcf9d --- /dev/null +++ b/generation_config.json @@ -0,0 +1,6 @@ +{ + "_from_model_config": true, + "bos_token_id": 1, + "eos_token_id": 2, + "transformers_version": "4.45.1" +} diff --git a/model-00001-of-00003.safetensors b/model-00001-of-00003.safetensors new file mode 100644 index 0000000..44281e1 --- /dev/null +++ b/model-00001-of-00003.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:694ff3011a5b8627ced5f199c9db28f9959260df8451373bad528b1c054fb440 +size 4991999400 diff --git a/model-00002-of-00003.safetensors b/model-00002-of-00003.safetensors new file mode 100644 index 0000000..2e37b8f --- /dev/null +++ b/model-00002-of-00003.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e8e43e9bd5457a95b5a1af6bfb55e2d590fc1d003f62865899c7e02514b36ac7 +size 4982793136 diff --git a/model-00003-of-00003.safetensors b/model-00003-of-00003.safetensors new file mode 100644 index 0000000..556cc08 --- /dev/null +++ b/model-00003-of-00003.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5a6db63d7b84e4be46d19faf8c23fae527bfa9b2f762fc0e1447a57981902748 +size 4489216120 diff --git a/model.safetensors.index.json b/model.safetensors.index.json new file mode 100644 index 0000000..dff83a7 --- /dev/null +++ b/model.safetensors.index.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f29b60424b26ff1ce1fd56bfacd94d1d6ee9f21c2bc6b1e41272f35b37e0b458 +size 86499 diff --git a/preprocessor_config.json b/preprocessor_config.json new file mode 100644 index 0000000..3e916d7 --- /dev/null +++ b/preprocessor_config.json @@ -0,0 +1,27 @@ +{ + "do_convert_rgb": true, + "do_normalize": true, + "do_rescale": true, + "do_resize": true, + "image_mean": [ + 0.48145466, + 0.4578275, + 0.40821073 + ], + "image_processor_type": "PixtralImageProcessor", + "image_std": [ + 0.26862954, + 0.26130258, + 0.27577711 + ], + "patch_size": { + "height": 16, + "width": 16 + }, + "processor_class": "PixtralProcessor", + "resample": 3, + "rescale_factor": 0.00392156862745098, + "size": { + "longest_edge": 1024 + } +} diff --git a/processor_config.json b/processor_config.json new file mode 100644 index 0000000..c9b049d --- /dev/null +++ b/processor_config.json @@ -0,0 +1,7 @@ +{ + "image_break_token": "[IMG_BREAK]", + "image_end_token": "[IMG_END]", + "image_token": "[IMG]", + "patch_size": 16, + "processor_class": "PixtralProcessor" +} diff --git a/recipe.yaml b/recipe.yaml new file mode 100644 index 0000000..c29f880 --- /dev/null +++ b/recipe.yaml @@ -0,0 +1,6 @@ +DEFAULT_stage: + DEFAULT_modifiers: + QuantizationModifier: + ignore: ['re:.*lm_head', 're:multi_modal_projector.*', 're:vision_tower.*'] + targets: Linear + scheme: FP8_DYNAMIC diff --git a/special_tokens_map.json b/special_tokens_map.json new file mode 100644 index 0000000..451134b --- /dev/null +++ b/special_tokens_map.json @@ -0,0 +1,23 @@ +{ + "bos_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "unk_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } +} diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..984b06f --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fcdf3e6b96371b9a5c8673bdf2f1bc838b994b73a6fa995dca62196346aaddb5 +size 17077311 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..a358b24 --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dbbdab721b449369ed5394908165221239f41c8411872b15968782c5597bba43 +size 178722