From 59d228c16b9414af221d07688c78c54a545e1622 Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Mon, 29 Jun 2026 04:18:13 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: RedHatAI/Llama-3.1-8B-tldr Source: Original Platform --- .gitattributes | 36 ++++ README.md | 298 ++++++++++++++++++++++++++++++ config.json | 35 ++++ configuration.json | 1 + generation_config.json | 9 + inference_performance/latency.png | 3 + model-00001-of-00004.safetensors | 3 + model-00002-of-00004.safetensors | 3 + model-00003-of-00004.safetensors | 3 + model-00004-of-00004.safetensors | 3 + model.safetensors.index.json | 3 + special_tokens_map.json | 23 +++ tokenizer.json | 3 + tokenizer_config.json | 3 + 14 files changed, 426 insertions(+) create mode 100644 .gitattributes create mode 100644 README.md create mode 100644 config.json create mode 100644 configuration.json create mode 100644 generation_config.json create mode 100644 inference_performance/latency.png create mode 100644 model-00001-of-00004.safetensors create mode 100644 model-00002-of-00004.safetensors create mode 100644 model-00003-of-00004.safetensors create mode 100644 model-00004-of-00004.safetensors create mode 100644 model.safetensors.index.json create mode 100644 special_tokens_map.json create mode 100644 tokenizer.json create mode 100644 tokenizer_config.json diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..52373fe --- /dev/null +++ b/.gitattributes @@ -0,0 +1,36 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..3ced7ae --- /dev/null +++ b/README.md @@ -0,0 +1,298 @@ +--- +library_name: transformers +license: llama3.1 +base_model: meta-llama/Llama-3.1-8B +tags: +- generated_from_trainer +datasets: +- trl-lib/tldr +--- + +[Built with Axolotl](https://github.com/axolotl-ai-cloud/axolotl) + +# Llama-3.1-8B-tldr + +## Model Overview +- **Model Architecture:** LlamaForCausalLM + - **Input:** Text + - **Output:** Text +- **Release Date:** 05/29/2025 +- **Version:** 1.0 +- **Intended Use Cases:** This model is finetuned to summarize text in the style of Reddit posts. +- **Out-of-scope:** Use in any manner that violates applicable laws or regulations (including trade compliance laws). Use in any other way that is prohibited by the Acceptable Use Policy and Llama 3.1 Community License. +- **Model Developers:** Red Hat (Neural Magic) + +This model is a fine-tuned version of [meta-llama/Llama-3.1-8B](https://huggingface.co/meta-llama/Llama-3.1-8B) on the [trl-lib/tldr](https://huggingface.co/datasets/trl-lib/tldr) dataset. +This model obtains 0.366 BERTScore on the test set of trl-lib/tldr. + + +## Deployment + +This model can be deployed efficiently using [vLLM](https://docs.vllm.ai/en/latest/), as shown in the example below. + +Run the following command to start the vLLM server: +```bash +vllm serve RedHatAI/Llama-3.1-8B-tldr +``` + +Once your server is started, you can query the model using the OpenAI API: + +```python +from openai import OpenAI + +openai_api_key = "EMPTY" +openai_api_base = "http://localhost:8000/v1" +client = OpenAI( + api_key=openai_api_key, + base_url=openai_api_base, +) + +post=""" +SUBREDDIT: r/AI + +TITLE: Training sparse LLMs + +POST: Now you can use the llm-compressor integration to axolotl to train sparse LLMs! + +It's super easy to use. See the example in https://huggingface.co/RedHatAI/Sparse-Llama-3.1-8B-tldr-2of4. + +And there's more. You can run 2:4 sparse models on vLLM and get significant speedupts on Hopper GPUs! +""" + +prompt = f"Give a TL;DR of the following Reddit post.\n<|user|>{post}\nTL;DR:\n<|assistant|>\n" + +completion = client.completions.create( + model="RedHatAI/Llama-3.1-8B-tldr", + prompt=prompt, + max_tokens=256, +) +print("Completion result:", completion) +``` + +## Training + +
See axolotl config + +axolotl version: `0.10.0.dev0` +```yaml +base_model: meta-llama/Llama-3.1-8B + +load_in_8bit: false +load_in_4bit: false +strict: false + +datasets: + - path: trl-lib/tldr + type: + system_prompt: "Give a TL;DR of the following Reddit post." + field_system: system + field_instruction: prompt + field_output: completion + format: "<|user|>\n{instruction}\n<|assistant|>\n" + no_input_format: "<|user|>\n{instruction}\n<|assistant|>\n" + split: train + +sequence_len: 4096 +sample_packing: true +pad_to_sequence_len: true +eval_sample_packing: true + +torch.compile: true +gradient_accumulation_steps: 1 +micro_batch_size: 4 +num_epochs: 3 +optimizer: adamw_bnb_8bit +lr_scheduler: cosine +learning_rate: 1e-5 +max_grad_norm: 1 + +gradient_checkpointing: true +gradient_checkpointing_kwargs: + use_reentrant: false + +train_on_inputs: false +bf16: auto +fp16: +tf32: false + +early_stopping_patience: +resume_from_checkpoint: +logging_steps: 1 +flash_attention: true + +warmup_ratio: 0.05 +evals_per_epoch: 4 +val_set_size: 0.05 +save_strategy: "best" +save_total_limit: 1 +metric_for_best_model: "loss" + +debug: +deepspeed: +weight_decay: 0.0 +special_tokens: + pad_token: "<|end_of_text|>" + +seed: 0 + +plugins: + - axolotl.integrations.liger.LigerPlugin + +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +liger_layer_norm: true +liger_fused_linear_cross_entropy: true +``` + +

+ +## Training + +
Training hyperparameters + +The following hyperparameters were used during training: +- learning_rate: 1e-05 +- train_batch_size: 4 +- eval_batch_size: 4 +- seed: 0 +- distributed_type: multi-GPU +- num_devices: 8 +- total_train_batch_size: 32 +- total_eval_batch_size: 32 +- optimizer: Use adamw_bnb_8bit with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments +- lr_scheduler_type: cosine +- lr_scheduler_warmup_steps: 49 +- num_epochs: 3.0 + +

+ +
Training results + +| Training Loss | Epoch | Step | Validation Loss | +|:-------------:|:------:|:----:|:---------------:| +| 2.2572 | 0.0031 | 1 | 2.2288 | +| 1.7865 | 0.2508 | 82 | 1.7680 | +| 1.7257 | 0.5015 | 164 | 1.7567 | +| 1.7343 | 0.7523 | 246 | 1.7489 | +| 1.7688 | 1.0031 | 328 | 1.7441 | +| 1.6822 | 1.2538 | 410 | 1.7493 | +| 1.6085 | 1.5046 | 492 | 1.7480 | +| 1.6627 | 1.7554 | 574 | 1.7444 | +| 1.729 | 2.0061 | 656 | 1.7426 | +| 1.6149 | 2.2569 | 738 | 1.7540 | +| 1.6002 | 2.5076 | 820 | 1.7537 | +| 1.6573 | 2.7584 | 902 | 1.7526 | + +

+ +
Framework versions + +- Transformers 4.51.3 +- Pytorch 2.7.0+cu126 +- Datasets 3.5.1 +- Tokenizers 0.21.1 + +

+ +## Evaluation + +The model was evaluated on the test split of [trl-lib/tldr](https://huggingface.co/datasets/trl-lib/tldr) using the Neural Magic fork of [lm-evaluation-harness](https://github.com/neuralmagic/lm-evaluation-harness/tree/tldr) (tldr branch). +One can reproduce these results by using the following command: + +```bash +lm_eval --model vllm --model_args "pretrained=RedHatAI/Llama-3.1-8B-tldr,dtype=auto,add_bos_token=True" --batch-size auto --tasks tldr +``` + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Metric + Llama-3.1-8B-Instruct + Llama-3.1-8B-tldr + Sparse-Llama-3.1-8B-tldr-2of4
(this model) +
BERTScore + -0.230 + 0.366 + 0.366 +
ROUGE-1 + 0.059 + 0.362 + 0.357 +
ROUGE-2 + 0.018 + 0.144 + 0.141 +
ROUGE-Lsum + 0.051 + 0.306 + 0.304 +
+ +## Inference Performance + +We evaluated the inference performance of this model using the first 1,000 samples from the training set of the [trl-lib/tldr](https://huggingface.co/datasets/trl-lib/tldr) dataset. +Benchmarking was conducted with [vLLM](https://docs.vllm.ai/en/latest/) version `0.9.0.1` and [GuideLLM](https://github.com/neuralmagic/guidellm) version `0.2.1`. + +The figure below presents the **mean end-to-end latency per request** across varying request rates. +Results are shown for this model, as well as two variants: +- **Dense-quantized:** [Llama-3.1-8B-tldr-FP8-dynamic](https://huggingface.co/RedHatAI/Llama-3.1-8B-tldr-FP8-dynamic) +- **Sparse-quantized:** [Sparse-Llama-3.1-8B-tldr-2of4-FP8-dynamic](https://huggingface.co/RedHatAI/Sparse-Llama-3.1-8B-tldr-2of4-FP8-dynamic) + +![Latency](./inference_performance/latency.png) + + +
Reproduction instructions + +To replicate the benchmark: + +1. Generate a JSON file containing the first 1,000 training samples: +```python +from datasets import load_dataset +ds = load_dataset("trl-lib/tldr", split="train").take(1000) +ds.to_json("tldr_1000.json") +``` + +2. Start a vLLM server using your target model: +```bash +vllm serve RedHatAI/Llama-3.1-8B-tldr +``` + +3. Run the benchmark with GuideLLM: +``` +GUIDELLM__OPENAI__MAX_OUTPUT_TOKENS=128 guidellm benchmark --target "http://localhost:8000" --rate-type sweep --data tldr_1000.json +``` +> The average output length is approximately 30 tokens per sample. We capped the generation at 128 tokens to reduce performance skew from rare, unusually verbose completions. + +
+ + + diff --git a/config.json b/config.json new file mode 100644 index 0000000..7b0ff36 --- /dev/null +++ b/config.json @@ -0,0 +1,35 @@ +{ + "architectures": [ + "LlamaForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 128000, + "eos_token_id": 128001, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 4096, + "initializer_range": 0.02, + "intermediate_size": 14336, + "max_position_embeddings": 131072, + "mlp_bias": false, + "model_type": "llama", + "num_attention_heads": 32, + "num_hidden_layers": 32, + "num_key_value_heads": 8, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_scaling": { + "factor": 8.0, + "high_freq_factor": 4.0, + "low_freq_factor": 1.0, + "original_max_position_embeddings": 8192, + "rope_type": "llama3" + }, + "rope_theta": 500000.0, + "tie_word_embeddings": false, + "torch_dtype": "bfloat16", + "transformers_version": "4.51.3", + "use_cache": false, + "vocab_size": 128256 +} diff --git a/configuration.json b/configuration.json new file mode 100644 index 0000000..bbeeda1 --- /dev/null +++ b/configuration.json @@ -0,0 +1 @@ +{"framework": "pytorch", "task": "text-generation", "allow_remote": true} \ No newline at end of file diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..81e8cb7 --- /dev/null +++ b/generation_config.json @@ -0,0 +1,9 @@ +{ + "_from_model_config": true, + "bos_token_id": 128000, + "do_sample": true, + "eos_token_id": 128001, + "temperature": 0.6, + "top_p": 0.9, + "transformers_version": "4.51.3" +} diff --git a/inference_performance/latency.png b/inference_performance/latency.png new file mode 100644 index 0000000..2c03a43 --- /dev/null +++ b/inference_performance/latency.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2d5174cb551515597a0a4910bd02a745a8b96bcf4cd4b466a98e91275790c305 +size 32626 diff --git a/model-00001-of-00004.safetensors b/model-00001-of-00004.safetensors new file mode 100644 index 0000000..fe11c58 --- /dev/null +++ b/model-00001-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2cbbe9ec7f43735641e5e794886e5d210175e5769a7655afb1481e638d57199f +size 4976698672 diff --git a/model-00002-of-00004.safetensors b/model-00002-of-00004.safetensors new file mode 100644 index 0000000..125998d --- /dev/null +++ b/model-00002-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:52cac5c1f72f72beb73d417b164da65ec35c1daa1c3203aa0b1292630181ef5c +size 4999802720 diff --git a/model-00003-of-00004.safetensors b/model-00003-of-00004.safetensors new file mode 100644 index 0000000..87e8591 --- /dev/null +++ b/model-00003-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:22dca7cbb83f049ee095930a3782f067b4b32502779a43916f7922fee49f7464 +size 4915916176 diff --git a/model-00004-of-00004.safetensors b/model-00004-of-00004.safetensors new file mode 100644 index 0000000..d4296c8 --- /dev/null +++ b/model-00004-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0b031ebdd04767a8cfc9138e3f7bab600a14465fe2fbe22ee54f372b08661bea +size 1168138808 diff --git a/model.safetensors.index.json b/model.safetensors.index.json new file mode 100644 index 0000000..a054aad --- /dev/null +++ b/model.safetensors.index.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:146776fce3f6db1103aa6f249e65ee5544c5923ce6f971b092eee79aa6e5d37b +size 23950 diff --git a/special_tokens_map.json b/special_tokens_map.json new file mode 100644 index 0000000..e5b39b6 --- /dev/null +++ b/special_tokens_map.json @@ -0,0 +1,23 @@ +{ + "bos_token": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } +} diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..1c1d8d5 --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b +size 17209920 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..e91812f --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fb0b184bfd935cbe6f8290f1af424c17814fd24dfc5aaac3be9b0b674fe40631 +size 50560