commit 61f733f303900d97010d0e733c4858ee835edb4b Author: ModelHub XC Date: Tue Sep 8 04:54:12 2026 +0800 初始化项目，由ModelHub XC社区提供模型 Model: RedHatAI/gemma-3-1b-it-quantized.w8a8 Source: Original Platform diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..b9df9f1 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,37 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text + diff --git a/README.md b/README.md new file mode 100644 index 0000000..ef21c37 --- /dev/null +++ b/README.md @@ -0,0 +1,236 @@ +--- +tags: +- vllm +- vision +- w8a8 +license: gemma +base_model: google/gemma-3-1b-it +library_name: transformers +--- + +# gemma-3-1b-it-quantized.w8a8 + +## Model Overview +- **Model Architecture:** google/gemma-3-1b-it + - **Input:** Vision-Text + - **Output:** Text +- **Model Optimizations:** + - **Weight quantization:** INT8 + - **Activation quantization:** INT8 +- **Release Date:** 6/4/2025 +- **Version:** 1.0 +- **Model Developers:** RedHatAI + +Quantized version of [google/gemma-3-1b-it](https://huggingface.co/google/gemma-3-1b-it). + +### Model Optimizations + +This model was obtained by quantizing the weights of [google/gemma-3-1b-it](https://huggingface.co/google/gemma-3-1b-it) to INT8 data type, ready for inference with vLLM >= 0.8.0. + +## Deployment + +### Use with vLLM + +This model can be deployed efficiently using the [vLLM](https://docs.vllm.ai/en/latest/) backend, as shown in the example below. + +```python +from vllm.assets.image import ImageAsset +from vllm import LLM, SamplingParams + +# prepare model +llm = LLM( + model="RedHatAI/gemma-3-1b-it-quantized.w8a8", + trust_remote_code=True, + max_model_len=4096, + max_num_seqs=2, +) + +# prepare inputs +question = "What is the content of this image?" +inputs = { + "prompt": f"<|user|>\n<|image_1|>\n{question}<|end|>\n<|assistant|>\n", + "multi_modal_data": { + "image": ImageAsset("cherry_blossom").pil_image.convert("RGB") + }, +} + +# generate response +print("========== SAMPLE GENERATION ==============") +outputs = llm.generate(inputs, SamplingParams(temperature=0.2, max_tokens=64)) +print(f"PROMPT : {outputs[0].prompt}") +print(f"RESPONSE: {outputs[0].outputs[0].text}") +print("==========================================") +``` + +vLLM also supports OpenAI-compatible serving. See the [documentation](https://docs.vllm.ai/en/latest/) for more details. + +## Creation + +This model was created with [llm-compressor](https://github.com/vllm-project/llm-compressor) by running the code snippet below: + +
+ Model Creation Code + +```python +import base64 +from io import BytesIO +import torch +from datasets import load_dataset +from transformers import AutoProcessor, Gemma3ForCausalLM +from llmcompressor.modifiers.quantization import GPTQModifier +from llmcompressor.transformers import oneshot + + +# Load model. +model_id = "google/gemma-3-1b-it" +model = Gemma3ForCausalLM.from_pretrained( + model_id, + device_map="auto", + torch_dtype="auto", +) +processor = AutoTokenizer.from_pretrained(model_id, trust_remote_code=True) + +# Oneshot arguments +DATASET_ID = "neuralmagic/calibration" +DATASET_SPLIT = {"LLM": "train[:512]"} +NUM_CALIBRATION_SAMPLES = 512 +MAX_SEQUENCE_LENGTH = 2048 + +# Load dataset and preprocess. +ds = load_dataset(DATASET_ID, split=DATASET_SPLIT) +ds = ds.shuffle(seed=42) + +dampening_frac=0.01 + +def data_collator(batch): + assert len(batch) == 1, "Only batch size of 1 is supported for calibration" + item = batch[0] + collated = {} + import torch + + + for key, value in item.items(): + if isinstance(value, torch.Tensor): + collated[key] = value.unsqueeze(0) + elif isinstance(value, list) and isinstance(value[0][0], int): + # Handle tokenized inputs like input_ids, attention_mask + collated[key] = torch.tensor(value) + elif isinstance(value, list) and isinstance(value[0][0], float): + # Handle possible float sequences + collated[key] = torch.tensor(value) + elif isinstance(value, list) and isinstance(value[0][0], torch.Tensor): + # Handle batched image data (e.g., pixel_values as [C, H, W]) + collated[key] = torch.stack(value) # -> [1, C, H, W] + elif isinstance(value, torch.Tensor): + collated[key] = value + else: + print(f"[WARN] Unrecognized type in collator for key={key}, type={type(value)}") + + return collated + + + +# Recipe +recipe = [ + GPTQModifier( + targets="Linear", + ignore=["re:.*lm_head.*", "re:.*embed_tokens.*", "re:vision_tower.*", "re:multi_modal_projector.*"], + sequential_update=True, + sequential_targets=["Gemma3DecoderLayer"], + dampening_frac=dampening_frac, + ) +] + +SAVE_DIR=f"{model_id.split('/')[1]}-quantized.w8a8" + +# Perform oneshot +oneshot( + model=model, + tokenizer=model_id, + dataset=ds, + recipe=recipe, + max_seq_length=MAX_SEQUENCE_LENGTH, + num_calibration_samples=NUM_CALIBRATION_SAMPLES, + trust_remote_code_model=True, + data_collator=data_collator, + output_dir=SAVE_DIR +) +``` +
+ +## Evaluation + +The model was evaluated using [lm_evaluation_harness](https://github.com/neuralmagic/lm-evaluation-harness) for OpenLLM v1 text benchmark. The evaluations were conducted using the following commands: + +
+Evaluation Commands + +### OpenLLM v1 +``` +lm_eval \ + --model vllm \ + --model_args pretrained="",dtype=auto,add_bos_token=True,max_model_len=4096,tensor_parallel_size=,gpu_memory_utilization=0.8,enable_chunked_prefill=True,trust_remote_code=True,enforce_eager=True \ + --tasks openllm \ + --batch_size auto +``` +
+ + +### Accuracy + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
CategoryMetricgoogle/gemma-3-1b-itRedHatAI/gemma-3-1b-it-quantized.w8a8Recovery (%)
OpenLLM V1ARC Challenge36.86%36.43%98.84% +
GSM8K25.17%24.87%98.80% +
Hellaswag56.03%55.62%99.25%
MMLU39.99%39.35%98.38%
Truthfulqa (mc2)38.54%38.22%99.17%
Winogrande58.88%58.96%100.13%
Average Score42.58%42.24%99.20%
diff --git a/added_tokens.json b/added_tokens.json new file mode 100644 index 0000000..e17bde0 --- /dev/null +++ b/added_tokens.json @@ -0,0 +1,3 @@ +{ + "": 262144 +} diff --git a/config.json b/config.json new file mode 100644 index 0000000..88cba9c --- /dev/null +++ b/config.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eec4001992e7b04e5d27e6d6d72d7a015ba2df3187e1883aae7e2faa885d4a12 +size 1945 diff --git a/configuration.json b/configuration.json new file mode 100644 index 0000000..bbeeda1 --- /dev/null +++ b/configuration.json @@ -0,0 +1 @@ +{"framework": "pytorch", "task": "text-generation", "allow_remote": true} \ No newline at end of file diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..c4400dd --- /dev/null +++ b/generation_config.json @@ -0,0 +1,13 @@ +{ + "bos_token_id": 2, + "cache_implementation": "hybrid", + "do_sample": true, + "eos_token_id": [ + 1, + 106 + ], + "pad_token_id": 0, + "top_k": 64, + "top_p": 0.95, + "transformers_version": "4.51.3" +} diff --git a/model.safetensors b/model.safetensors new file mode 100644 index 0000000..a92a119 --- /dev/null +++ b/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8cc0227db20ae63a50c9c1edd6dda3fa0b92a4efca9ad9ca7dbb396b35cae5c1 +size 1906969248 diff --git a/recipe.yaml b/recipe.yaml new file mode 100644 index 0000000..d9838bf --- /dev/null +++ b/recipe.yaml @@ -0,0 +1,12 @@ +quant_stage: + quant_modifiers: + GPTQModifier: + dampening_frac: 0.01 + ignore: ['re:.*lm_head.*', 're:.*embed_tokens.*', 're:vision_tower.*', 're:multi_modal_projector.*'] + sequential_update: true + config_groups: + group_0: + targets: [Linear] + weights: {num_bits: 8, type: int, symmetric: true, strategy: channel, observer: mse} + input_activations: {num_bits: 8, type: int, symmetric: true, strategy: token, dynamic: true, + observer: memoryless} diff --git a/special_tokens_map.json b/special_tokens_map.json new file mode 100644 index 0000000..1a61932 --- /dev/null +++ b/special_tokens_map.json @@ -0,0 +1,33 @@ +{ + "boi_token": "", + "bos_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eoi_token": "", + "eos_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "image_token": "", + "pad_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "unk_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } +} diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..29401f9 --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4667f2089529e8e7657cfb6d1c19910ae71ff5f28aa7ab2ff2763330affad795 +size 33384568 diff --git a/tokenizer.model b/tokenizer.model new file mode 100644 index 0000000..14f810a --- /dev/null +++ b/tokenizer.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1299c11d7cf632ef3b4e11937501358ada021bbdf7c47638d13c0ee982f2e79c +size 4689074 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..a6cc152 --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bfe25c2735e395407beb78456ea9a6984a1f00d8c16fa04a8b75f2a614cf53e1 +size 1156999