From 3b4ff912d188a58f42290ad42071f1d32fb832e9 Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Tue, 3 Nov 2026 13:37:19 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: RightNowAI/RightNow-Arabic-0.5B-Turbo Source: Original Platform --- .gitattributes | 44 ++++ README.md | 235 ++++++++++++++++++++ benchmark_chart_data.json | 82 +++++++ benchmark_data.json | 73 ++++++ benchmark_pareto.png | 3 + benchmarks.json | 48 ++++ bf16/chat_template.jinja | 54 +++++ bf16/config.json | 58 +++++ bf16/generation_config.json | 15 ++ bf16/model.safetensors | 3 + bf16/tokenizer.json | 3 + bf16/tokenizer_config.json | 15 ++ gguf/RightNow-Arabic-0.5B-Turbo-f16.gguf | 3 + gguf/RightNow-Arabic-0.5B-Turbo-q4_k_m.gguf | 3 + gguf/RightNow-Arabic-0.5B-Turbo-q5_k_m.gguf | 3 + gguf/RightNow-Arabic-0.5B-Turbo-q8_0.gguf | 3 + inference.json | 11 + int8/chat_template.jinja | 54 +++++ int8/config.json | 73 ++++++ int8/generation_config.json | 9 + int8/model.safetensors | 3 + int8/tokenizer.json | 3 + int8/tokenizer_config.json | 29 +++ speed_comparison.json | 106 +++++++++ tokenizer_fertility.json | 10 + 25 files changed, 943 insertions(+) create mode 100644 .gitattributes create mode 100644 README.md create mode 100644 benchmark_chart_data.json create mode 100644 benchmark_data.json create mode 100644 benchmark_pareto.png create mode 100644 benchmarks.json create mode 100644 bf16/chat_template.jinja create mode 100644 bf16/config.json create mode 100644 bf16/generation_config.json create mode 100644 bf16/model.safetensors create mode 100644 bf16/tokenizer.json create mode 100644 bf16/tokenizer_config.json create mode 100644 gguf/RightNow-Arabic-0.5B-Turbo-f16.gguf create mode 100644 gguf/RightNow-Arabic-0.5B-Turbo-q4_k_m.gguf create mode 100644 gguf/RightNow-Arabic-0.5B-Turbo-q5_k_m.gguf create mode 100644 gguf/RightNow-Arabic-0.5B-Turbo-q8_0.gguf create mode 100644 inference.json create mode 100644 int8/chat_template.jinja create mode 100644 int8/config.json create mode 100644 int8/generation_config.json create mode 100644 int8/model.safetensors create mode 100644 int8/tokenizer.json create mode 100644 int8/tokenizer_config.json create mode 100644 speed_comparison.json create mode 100644 tokenizer_fertility.json diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..2f2c46e --- /dev/null +++ b/.gitattributes @@ -0,0 +1,44 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +benchmark_pareto.png filter=lfs diff=lfs merge=lfs -text +benchmark_bars.png filter=lfs diff=lfs merge=lfs -text +benchmark_chart.png filter=lfs diff=lfs merge=lfs -text +bf16/tokenizer.json filter=lfs diff=lfs merge=lfs -text +int8/tokenizer.json filter=lfs diff=lfs merge=lfs -text +gguf/RightNow-Arabic-0.5B-Turbo-f16.gguf filter=lfs diff=lfs merge=lfs -text +gguf/RightNow-Arabic-0.5B-Turbo-q4_k_m.gguf filter=lfs diff=lfs merge=lfs -text +gguf/RightNow-Arabic-0.5B-Turbo-q5_k_m.gguf filter=lfs diff=lfs merge=lfs -text +gguf/RightNow-Arabic-0.5B-Turbo-q8_0.gguf filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..5cedc2c --- /dev/null +++ b/README.md @@ -0,0 +1,235 @@ +--- +license: apache-2.0 +language: +- ar +- en +base_model: Qwen/Qwen2.5-0.5B +pipeline_tag: text-generation +tags: +- arabic +- edge +- small-language-model +- sft +- dpo +- gguf +- qwen2 +library_name: transformers +model-index: +- name: RightNow-Arabic-0.5B-Turbo + results: + - task: + type: text-generation + name: COPA-ar + dataset: + name: copa_ar + type: copa_ar + metrics: + - type: acc_norm + value: 58.4 + name: Accuracy (norm) + - task: + type: text-generation + name: Arabic HellaSwag + dataset: + name: arabic_mt_hellaswag + type: arabic_mt_hellaswag + metrics: + - type: acc_norm + value: 26.0 + name: Accuracy (norm) + - task: + type: text-generation + name: ArabicMMLU + dataset: + name: arabic_leaderboard_arabic_mmlu + type: arabic_leaderboard_arabic_mmlu + metrics: + - type: acc + value: 23.2 + name: Accuracy +--- + +
+ +# RightNow-Arabic-0.5B-Turbo + +### The smallest open Arabic-specialized decoder LLM + +**518M parameters | 398 MB on disk (q4_k_m) | 635 tok/s on H100** + +*Built by [RightNow AI](https://rightnowai.co)* + + + +
+ +--- + +## What is this? + +RightNow-Arabic-0.5B-Turbo is a **518M-parameter Arabic-specialized language model** built on top of Qwen2.5-0.5B via vocabulary injection, continued pretraining, supervised fine-tuning, and direct preference optimization. It is the **smallest open Arabic-specialized decoder LLM** with publicly available weights. + +The model targets **edge deployment**: phones, laptops, embedded devices, and browsers. Quantized to q4_k_m it fits in 398 MB and generates 635 tokens/s at batch size 1. + +## Key Features + +- **27,032 new Arabic tokens** added via mean-subtoken initialization, cutting Arabic tokenizer fertility by 17.3% (2.18 to 1.80 tokens/word) +- **504M Arabic pretraining tokens** (Arabic Wikipedia) on 8xH100 SXM5 with FSDP + FlashAttention varlen + Liger fused kernels +- **129,116 Arabic instruction pairs** for SFT with response-only loss masking +- **6,750 Arabic preference pairs** for DPO +- **Weight soup merging** (DPO 50%, SFT 25%, Pretrain 25%) for optimal accuracy +- **4 GGUF quantizations** for instant edge deployment + +## Benchmarks + +Evaluated with [lm-evaluation-harness](https://github.com/EleutherAI/lm-evaluation-harness) v0.4.11, `apply_chat_template=True`, `limit=200`, `acc_norm` preferred. + +### Head-to-head comparison + +| Model | Params | COPA-ar | HellaSwag-ar | ArabicMMLU | Mean | +|-------|--------|---------|-------------|------------|------| +| **RightNow-Arabic-0.5B-Turbo (ours)** | **518M** | **58.4%** | **26.0%** | 23.2% | **35.9%** | +| Qwen2.5-0.5B-Instruct | 494M | 53.9% | 22.5% | **26.0%** | 34.1% | +| Falcon-H1-0.5B-Instruct | 524M | 44.9% | 23.0% | 24.2% | 30.7% | +| Falcon-H1-1.5B-Instruct | 1.5B | 58.4% | 27.5% | 32.7% | 39.5% | +| AceGPT-7B-chat | 7B | 69.7% | 27.0% | 35.0% | 43.9% | +| ALLaM-7B-Instruct | 7B | 68.5% | 29.0% | 52.2% | 49.9% | +| SILMA-9B-Instruct | 9B | 69.7% | 38.0% | 52.9% | 53.5% | + +**Among 0.5B models**: best on COPA-ar (+4.5 vs Qwen), best on HellaSwag-ar (+3.5 vs Qwen), best mean (+1.8 vs Qwen, +5.2 vs Falcon). + +**Ties Falcon-H1-1.5B** on COPA-ar (both 58.4%) at one-third the parameters. + +**Recovers 67%** of SILMA-9B mean accuracy at 5.8% of the parameters. + +## Available Formats + +| Format | Size | Speed (tok/s, bs=1, H100) | Use case | +|--------|------|---------------------------|----------| +| bf16 | 1.04 GB | 82 (HF generate) | Fine-tuning, research | +| int8 | 664 MB | -- | Reduced memory inference | +| GGUF f16 | 988 MB | 582 | Maximum quality | +| **GGUF q8_0** | **525 MB** | **646** | Best speed | +| GGUF q5_k_m | 419 MB | 634 | Balanced | +| **GGUF q4_k_m** | **398 MB** | **635** | Smallest footprint | + +## Quick Start + +### With Transformers (bf16) + +```python +from transformers import AutoModelForCausalLM, AutoTokenizer + +model_id = "RightNowAI/RightNow-Arabic-0.5B-Turbo" +tokenizer = AutoTokenizer.from_pretrained(model_id, subfolder="bf16") +model = AutoModelForCausalLM.from_pretrained( + model_id, subfolder="bf16", + torch_dtype="bfloat16", device_map="auto" +) + +messages = [ + {"role": "system", "content": "أنت مساعد ذكي يجيب باللغة العربية الفصحى"}, + {"role": "user", "content": "ما هي عاصمة المملكة العربية السعودية؟"} +] +text = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True) +inputs = tokenizer(text, return_tensors="pt").to(model.device) +outputs = model.generate(**inputs, max_new_tokens=256) +print(tokenizer.decode(outputs[0][inputs.input_ids.shape[1]:], skip_special_tokens=True)) +``` + +### With llama.cpp (GGUF) + +```bash +# Download the q8_0 quantization (best speed) +huggingface-cli download RightNowAI/RightNow-Arabic-0.5B-Turbo \ + gguf/RightNow-Arabic-0.5B-Turbo-q8_0.gguf --local-dir . + +# Run inference +./llama-cli -m RightNow-Arabic-0.5B-Turbo-q8_0.gguf \ + -p "ما هي أكبر مدينة في مصر؟" \ + -n 128 --temp 0.7 +``` + +## Training Pipeline + +``` +Qwen2.5-0.5B (494M, 151,665 vocab) + | + v +Tokenizer Surgery (+27,032 Arabic tokens -> 178,697 vocab) + - SentencePiece unigram 32k on 12.5 GB Arabic corpus + - Mean-subtoken embedding initialization + - Fertility: 2.18 -> 1.80 tok/word (-17.3%) + | + v +Continued Pretraining (504M arwiki tokens) + - 2,500 steps, 8xH100 SXM5 + - FSDP _HYBRID_SHARD_ZERO2 + FlashAttention varlen + Liger + - Loss: 14.21 -> 1.69 | Wall time: 6h 57m + | + v +Supervised Fine-Tuning (129,116 instructions) + - 5 datasets: evol-instruct-arabic, alpaca-gpt4-arabic, + sharegpt-arabic, CIDAR, aya_dataset + - Response-only loss masking (72.1% of tokens carry loss) + - 5 epochs, 418 steps | Wall time: 12m + | + v +Direct Preference Optimization (6,750 pairs) + - argilla-dpo-mix-7k-arabic + - 2 epochs, 844 steps | Wall time: 34m + | + v +Weight Soup Merging + - Linear(DPO 0.5, SFT 0.25, Pretrain 0.25) + - +0.44 points mean accuracy over DPO alone + | + v +Export: bf16, int8, GGUF {f16, q8_0, q5_k_m, q4_k_m} +``` + +## Training Data + +| Dataset | Examples/Tokens | Use | +|---------|----------------|-----| +| Arabic Wikipedia (wikimedia/wikipedia 20231101.ar) | 504M tokens | Continued pretraining | +| FreedomIntelligence/evol-instruct-arabic | 59,022 | SFT | +| FreedomIntelligence/alpaca-gpt4-arabic | 49,969 | SFT | +| FreedomIntelligence/sharegpt-arabic | 5,231 | SFT | +| arbml/CIDAR | 10,000 | SFT | +| CohereForAI/aya_dataset (Arabic) | 4,947 | SFT | +| 2A2I/argilla-dpo-mix-7k-arabic | 6,750 | DPO | + +## Limitations + +- **Knowledge ceiling**: At 518M parameters, ArabicMMLU-style knowledge tasks lag 7B+ models by 12-30 points. This is a parameter-count limit, not a training limit. +- **MSA only**: Trained on Wikipedia (Modern Standard Arabic). Dialects (Egyptian, Gulf, Levantine) get MSA responses. +- **504M pretraining tokens**: Below Chinchilla-optimal ratio. More Arabic data would improve knowledge tasks. +- **DPO was weak**: 6,750 machine-translated preference pairs provided minimal signal at 0.5B scale. The weight soup merge was more impactful. +- **GGUF tile alignment**: q4_k_m and q5_k_m fall back to higher-bit quantization for 144/290 tensors due to the expanded vocabulary not aligning with k-quant tile sizes. + +## Hardware + +All training ran on a single Nebius `gpu-h100-sxm` node: +- 8x NVIDIA H100 80 GB SXM5 HBM3, NVLink4 +- 128 vCPUs, 1.5 TiB RAM +- CUDA 13.0, PyTorch 2.11, flash-attn 2.8.3, transformers 5.5.0 + +## Citation + +```bibtex +@article{jaber2025rightnow, + title={RightNow-Arabic-0.5B-Turbo: An Open Sub-1B Arabic Language Model via Vocabulary Injection and Edge-First Deployment}, + author={Jaber, Jaber and Jaber, Osama}, + year={2025}, + url={https://huggingface.co/RightNowAI/RightNow-Arabic-0.5B-Turbo} +} +``` + +## License + +Apache 2.0 (same as the base Qwen2.5-0.5B model). + +--- + +*Built by [RightNow AI](https://rightnowai.co)* diff --git a/benchmark_chart_data.json b/benchmark_chart_data.json new file mode 100644 index 0000000..b60382e --- /dev/null +++ b/benchmark_chart_data.json @@ -0,0 +1,82 @@ +{ + "source": "/home/ubuntu/rightnow-arabic-b-turbo/scripts/make_chart.py", + "data": [ + { + "model": "RightNow-Arabic-0.5B-Turbo", + "params_B": 0.518, + "mean_acc_pct": 35.9, + "disk_gb_best_quant": 0.4, + "category": "ours" + }, + { + "model": "Qwen2.5-0.5B-Instruct", + "params_B": 0.494, + "mean_acc_pct": 34.1, + "disk_gb_best_quant": 0.5, + "category": "same_class" + }, + { + "model": "Falcon-H1-0.5B-Instruct", + "params_B": 0.524, + "mean_acc_pct": 30.7, + "disk_gb_best_quant": 0.55, + "category": "same_class" + }, + { + "model": "Falcon-H1-1.5B-Instruct", + "params_B": 1.5, + "mean_acc_pct": 39.5, + "disk_gb_best_quant": 1.5, + "category": "bigger_open" + }, + { + "model": "AceGPT-7B-chat", + "params_B": 7.0, + "mean_acc_pct": 43.9, + "disk_gb_best_quant": 4.0, + "category": "bigger_open" + }, + { + "model": "ALLaM-7B-Instruct", + "params_B": 7.0, + "mean_acc_pct": 49.9, + "disk_gb_best_quant": 4.0, + "category": "bigger_open" + }, + { + "model": "SILMA-9B-Instruct", + "params_B": 9.0, + "mean_acc_pct": 53.5, + "disk_gb_best_quant": 5.0, + "category": "bigger_open" + }, + { + "model": "Jais-13B-chat", + "params_B": 13.0, + "mean_acc_pct": 50.0, + "disk_gb_best_quant": 7.0, + "category": "bigger_open" + }, + { + "model": "Kuwain-1.5B (paper only)", + "params_B": 1.5, + "mean_acc_pct": 44.0, + "disk_gb_best_quant": 0.8, + "category": "closed" + }, + { + "model": "Falcon-Arabic-7B (closed)", + "params_B": 7.0, + "mean_acc_pct": 52.0, + "disk_gb_best_quant": 4.0, + "category": "closed" + } + ], + "benchmarks": [ + "copa_ar", + "arabic_mt_hellaswag", + "arabic_leaderboard_arabic_mmlu" + ], + "methodology": "lm-eval-harness v0.4.11, apply_chat_template=True (where available), limit=200, acc_norm preferred over acc", + "note": "Kuwain-1.5B and Falcon-Arabic-7B use estimated numbers from their papers \u2014 weights are not publicly available for direct evaluation." +} \ No newline at end of file diff --git a/benchmark_data.json b/benchmark_data.json new file mode 100644 index 0000000..fc0df01 --- /dev/null +++ b/benchmark_data.json @@ -0,0 +1,73 @@ +{ + "methodology": "lm-evaluation-harness v0.4.11, apply_chat_template=True, limit=200, acc_norm preferred", + "tasks": [ + "copa_ar", + "arabic_mt_hellaswag", + "arabic_leaderboard_arabic_mmlu" + ], + "models": [ + { + "name": "RightNow-Arabic-0.5B-Turbo", + "params_B": 0.518, + "copa_ar": 58.4, + "arabic_mt_hellaswag": 26.0, + "arabic_leaderboard_arabic_mmlu": 23.2, + "mean": 35.87, + "category": "ours" + }, + { + "name": "Qwen2.5-0.5B-Instruct", + "params_B": 0.494, + "copa_ar": 53.9, + "arabic_mt_hellaswag": 22.5, + "arabic_leaderboard_arabic_mmlu": 26.0, + "mean": 34.13, + "category": "small" + }, + { + "name": "Falcon-H1-0.5B-Instruct", + "params_B": 0.524, + "copa_ar": 44.9, + "arabic_mt_hellaswag": 23.0, + "arabic_leaderboard_arabic_mmlu": 24.2, + "mean": 30.7, + "category": "small" + }, + { + "name": "Falcon-H1-1.5B-Instruct", + "params_B": 1.5, + "copa_ar": 58.4, + "arabic_mt_hellaswag": 27.5, + "arabic_leaderboard_arabic_mmlu": 32.7, + "mean": 39.53, + "category": "medium" + }, + { + "name": "AceGPT-7B-chat", + "params_B": 7.0, + "copa_ar": 69.7, + "arabic_mt_hellaswag": 27.0, + "arabic_leaderboard_arabic_mmlu": 35.0, + "mean": 43.9, + "category": "large" + }, + { + "name": "ALLaM-7B-Instruct", + "params_B": 7.0, + "copa_ar": 68.5, + "arabic_mt_hellaswag": 29.0, + "arabic_leaderboard_arabic_mmlu": 52.2, + "mean": 49.9, + "category": "large" + }, + { + "name": "SILMA-9B-Instruct", + "params_B": 9.0, + "copa_ar": 69.7, + "arabic_mt_hellaswag": 38.0, + "arabic_leaderboard_arabic_mmlu": 52.9, + "mean": 53.53, + "category": "xlarge" + } + ] +} \ No newline at end of file diff --git a/benchmark_pareto.png b/benchmark_pareto.png new file mode 100644 index 0000000..d13dc2f --- /dev/null +++ b/benchmark_pareto.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:891e54f4c0595e5798fb98124b8a117b4e75afc453544cd6e9473738a1fd0901 +size 198855 diff --git a/benchmarks.json b/benchmarks.json new file mode 100644 index 0000000..473798a --- /dev/null +++ b/benchmarks.json @@ -0,0 +1,48 @@ +{ + "final": { + "copa_ar": { + "alias": "copa_ar", + "acc,none": 0.5280898876404494, + "acc_stderr,none": 0.05321600047878916, + "acc_norm,none": 0.5842696629213483, + "acc_norm_stderr,none": 0.05253771631269923 + }, + "arabic_mt_hellaswag": { + "alias": "arabic_mt_hellaswag", + "acc,none": 0.195, + "acc_stderr,none": 0.028085923439997246, + "acc_norm,none": 0.26, + "acc_norm_stderr,none": 0.031093957143700304 + }, + "arabic_leaderboard_arabic_mmlu": { + "acc,none": 0.23238593052379397, + "acc_stderr,none": 0.0043980977605237925, + "acc_norm,none": 0.23238593052379397, + "acc_norm_stderr,none": 0.0043980977605237925, + "alias": "arabic_leaderboard_arabic_mmlu" + } + }, + "baseline": { + "copa_ar": { + "alias": "copa_ar", + "acc,none": 0.5617977528089888, + "acc_stderr,none": 0.05289150964557706, + "acc_norm,none": 0.5393258426966292, + "acc_norm_stderr,none": 0.05313506354909072 + }, + "arabic_mt_hellaswag": { + "alias": "arabic_mt_hellaswag", + "acc,none": 0.225, + "acc_stderr,none": 0.029601626330440618, + "acc_norm,none": 0.225, + "acc_norm_stderr,none": 0.029601626330440618 + }, + "arabic_leaderboard_arabic_mmlu": { + "acc,none": 0.2598279429380377, + "acc_stderr,none": 0.004566720578133093, + "acc_norm,none": 0.2598279429380377, + "acc_norm_stderr,none": 0.004566720578133093, + "alias": "arabic_leaderboard_arabic_mmlu" + } + } +} \ No newline at end of file diff --git a/bf16/chat_template.jinja b/bf16/chat_template.jinja new file mode 100644 index 0000000..28028c0 --- /dev/null +++ b/bf16/chat_template.jinja @@ -0,0 +1,54 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0]['role'] == 'system' %} + {{- messages[0]['content'] }} + {%- else %} + {{- 'You are a helpful assistant.' }} + {%- endif %} + {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0]['role'] == 'system' %} + {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }} + {%- else %} + {{- '<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {{- '<|im_start|>' + message.role }} + {%- if message.content %} + {{- '\n' + message.content }} + {%- endif %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {{- tool_call.arguments | tojson }} + {{- '}\n' }} + {%- endfor %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} +{%- endif %} diff --git a/bf16/config.json b/bf16/config.json new file mode 100644 index 0000000..84abf2a --- /dev/null +++ b/bf16/config.json @@ -0,0 +1,58 @@ +{ + "architectures": [ + "Qwen2ForCausalLM" + ], + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "bfloat16", + "eos_token_id": 151643, + "hidden_act": "silu", + "hidden_size": 896, + "initializer_range": 0.02, + "intermediate_size": 4864, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 32768, + "max_window_layers": 24, + "model_type": "qwen2", + "num_attention_heads": 14, + "num_hidden_layers": 24, + "num_key_value_heads": 2, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000.0, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": true, + "transformers_version": "5.5.0", + "use_cache": false, + "use_mrope": false, + "use_sliding_window": false, + "vocab_size": 178697 +} diff --git a/bf16/generation_config.json b/bf16/generation_config.json new file mode 100644 index 0000000..31492ca --- /dev/null +++ b/bf16/generation_config.json @@ -0,0 +1,15 @@ +{ + "do_sample": true, + "eos_token_id": [ + 151643, + 151645 + ], + "max_new_tokens": 512, + "no_repeat_ngram_size": 4, + "pad_token_id": 151643, + "repetition_penalty": 1.15, + "temperature": 0.7, + "top_k": 50, + "top_p": 0.9, + "transformers_version": "5.5.0" +} diff --git a/bf16/model.safetensors b/bf16/model.safetensors new file mode 100644 index 0000000..2ec6c84 --- /dev/null +++ b/bf16/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1dfdfea7ba4fdc11c1aeb53bbcc05e9c981901ecc4efa0c9b745aec2474132c9 +size 1036053584 diff --git a/bf16/tokenizer.json b/bf16/tokenizer.json new file mode 100644 index 0000000..440c058 --- /dev/null +++ b/bf16/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f64f070635ecc8594a491b9bd7d8be78908632011d0ed1a800618c15aab3ac43 +size 16507759 diff --git a/bf16/tokenizer_config.json b/bf16/tokenizer_config.json new file mode 100644 index 0000000..73bbf27 --- /dev/null +++ b/bf16/tokenizer_config.json @@ -0,0 +1,15 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": {}, + "is_local": true, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} \ No newline at end of file diff --git a/gguf/RightNow-Arabic-0.5B-Turbo-f16.gguf b/gguf/RightNow-Arabic-0.5B-Turbo-f16.gguf new file mode 100644 index 0000000..c40295b --- /dev/null +++ b/gguf/RightNow-Arabic-0.5B-Turbo-f16.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f9b6512bea41870a7f348c8d16142d2b064ae697215421d90317f38b5f9f1458 +size 1042731232 diff --git a/gguf/RightNow-Arabic-0.5B-Turbo-q4_k_m.gguf b/gguf/RightNow-Arabic-0.5B-Turbo-q4_k_m.gguf new file mode 100644 index 0000000..f57f15b --- /dev/null +++ b/gguf/RightNow-Arabic-0.5B-Turbo-q4_k_m.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:03a58f3cb23af68dbe4247ac2f3f67fdeb8bc3df8bb7e41e0ac0049f8b5211a1 +size 423903072 diff --git a/gguf/RightNow-Arabic-0.5B-Turbo-q5_k_m.gguf b/gguf/RightNow-Arabic-0.5B-Turbo-q5_k_m.gguf new file mode 100644 index 0000000..05ad798 --- /dev/null +++ b/gguf/RightNow-Arabic-0.5B-Turbo-q5_k_m.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f3e9c1aea58fa42105faf049cbf5c61a14503f97a09177668ac9c1a66bc6d15 +size 446181216 diff --git a/gguf/RightNow-Arabic-0.5B-Turbo-q8_0.gguf b/gguf/RightNow-Arabic-0.5B-Turbo-q8_0.gguf new file mode 100644 index 0000000..cb0a743 --- /dev/null +++ b/gguf/RightNow-Arabic-0.5B-Turbo-q8_0.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:26affe6dda8199e38512d912b2a2dc2e4e388cd823ee7d2d8e926b45085692fe +size 557163360 diff --git a/inference.json b/inference.json new file mode 100644 index 0000000..31d2795 --- /dev/null +++ b/inference.json @@ -0,0 +1,11 @@ +{ + "_method": "HF generate(), greedy, attn_implementation=sdpa, no repetition penalty, no n-gram blocking", + "_note": "vLLM/TGI/SGLang typically achieve 2-5x higher tok/s at small batches due to CUDA graphs and optimized kernels. Use vLLM for production deployment.", + "bs1_tok_per_sec": 82, + "bs4_tok_per_sec": 323, + "bs8_tok_per_sec": 618, + "bs32_tok_per_sec": 2154, + "ttft_ms_bs1": 14, + "peak_mem_gb_bf16": 1.17, + "peak_mem_gb_int8": 0.66 +} \ No newline at end of file diff --git a/int8/chat_template.jinja b/int8/chat_template.jinja new file mode 100644 index 0000000..28028c0 --- /dev/null +++ b/int8/chat_template.jinja @@ -0,0 +1,54 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0]['role'] == 'system' %} + {{- messages[0]['content'] }} + {%- else %} + {{- 'You are a helpful assistant.' }} + {%- endif %} + {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0]['role'] == 'system' %} + {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }} + {%- else %} + {{- '<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {{- '<|im_start|>' + message.role }} + {%- if message.content %} + {{- '\n' + message.content }} + {%- endif %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {{- tool_call.arguments | tojson }} + {{- '}\n' }} + {%- endfor %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} +{%- endif %} diff --git a/int8/config.json b/int8/config.json new file mode 100644 index 0000000..b3d8b55 --- /dev/null +++ b/int8/config.json @@ -0,0 +1,73 @@ +{ + "architectures": [ + "Qwen2ForCausalLM" + ], + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "bfloat16", + "eos_token_id": 151643, + "hidden_act": "silu", + "hidden_size": 896, + "initializer_range": 0.02, + "intermediate_size": 4864, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 32768, + "max_window_layers": 24, + "model_type": "qwen2", + "num_attention_heads": 14, + "num_hidden_layers": 24, + "num_key_value_heads": 2, + "pad_token_id": 151643, + "quantization_config": { + "_load_in_4bit": false, + "_load_in_8bit": true, + "bnb_4bit_compute_dtype": "float32", + "bnb_4bit_quant_storage": "uint8", + "bnb_4bit_quant_type": "fp4", + "bnb_4bit_use_double_quant": false, + "llm_int8_enable_fp32_cpu_offload": false, + "llm_int8_has_fp16_weight": false, + "llm_int8_skip_modules": null, + "llm_int8_threshold": 6.0, + "load_in_4bit": false, + "load_in_8bit": true, + "quant_method": "bitsandbytes" + }, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000.0, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": true, + "transformers_version": "5.5.0", + "use_cache": false, + "use_mrope": false, + "use_sliding_window": false, + "vocab_size": 178697 +} diff --git a/int8/generation_config.json b/int8/generation_config.json new file mode 100644 index 0000000..2429260 --- /dev/null +++ b/int8/generation_config.json @@ -0,0 +1,9 @@ +{ + "do_sample": false, + "eos_token_id": [ + 151643 + ], + "max_new_tokens": 2048, + "pad_token_id": 151643, + "transformers_version": "5.5.0" +} diff --git a/int8/model.safetensors b/int8/model.safetensors new file mode 100644 index 0000000..16f047b --- /dev/null +++ b/int8/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:38232218af24b501cc44d3d724e3c363f7b93371df8dce75ed49737c1c647035 +size 679478360 diff --git a/int8/tokenizer.json b/int8/tokenizer.json new file mode 100644 index 0000000..440c058 --- /dev/null +++ b/int8/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f64f070635ecc8594a491b9bd7d8be78908632011d0ed1a800618c15aab3ac43 +size 16507759 diff --git a/int8/tokenizer_config.json b/int8/tokenizer_config.json new file mode 100644 index 0000000..1ce8d7b --- /dev/null +++ b/int8/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": true, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/speed_comparison.json b/speed_comparison.json new file mode 100644 index 0000000..48e0d7a --- /dev/null +++ b/speed_comparison.json @@ -0,0 +1,106 @@ +{ + "current_quality": { + "bs1": { + "bs": 1, + "elapsed_s": 4.636457004991826, + "tokens_generated": 128, + "tok_per_sec": 27.607287172552066 + }, + "bs4": { + "bs": 4, + "elapsed_s": 4.483801496011438, + "tokens_generated": 492, + "tok_per_sec": 109.72831880217227 + }, + "bs8": { + "bs": 8, + "elapsed_s": 4.641314439999405, + "tokens_generated": 992, + "tok_per_sec": 213.73255633163416 + }, + "bs32": { + "bs": 32, + "elapsed_s": 4.989478911011247, + "tokens_generated": 3968, + "tok_per_sec": 795.2734285023328 + } + }, + "clean_greedy_sdpa": { + "bs1": { + "bs": 1, + "elapsed_s": 1.5616609920107294, + "tokens_generated": 128, + "tok_per_sec": 81.96401181487703 + }, + "bs4": { + "bs": 4, + "elapsed_s": 1.525149337016046, + "tokens_generated": 492, + "tok_per_sec": 322.5913607664137 + }, + "bs8": { + "bs": 8, + "elapsed_s": 1.604114955989644, + "tokens_generated": 992, + "tok_per_sec": 618.4095449618166 + }, + "bs32": { + "bs": 32, + "elapsed_s": 1.8422480699955486, + "tokens_generated": 3968, + "tok_per_sec": 2153.8901652965706 + } + }, + "clean_greedy_fa2": { + "bs1": { + "bs": 1, + "elapsed_s": 2.231158509996021, + "tokens_generated": 128, + "tok_per_sec": 57.36929914505638 + }, + "bs4": { + "bs": 4, + "elapsed_s": 1.9051841270120349, + "tokens_generated": 492, + "tok_per_sec": 258.2427561852619 + }, + "bs8": { + "bs": 8, + "elapsed_s": 1.9795321589917876, + "tokens_generated": 984, + "tok_per_sec": 497.08715038060785 + }, + "bs32": { + "bs": 32, + "elapsed_s": 2.203783990989905, + "tokens_generated": 3936, + "tok_per_sec": 1786.0189637878307 + } + }, + "compiled_fa2": { + "bs1": { + "bs": 1, + "elapsed_s": 2.224584960989887, + "tokens_generated": 128, + "tok_per_sec": 57.538822856665846 + }, + "bs4": { + "bs": 4, + "elapsed_s": 1.940974594996078, + "tokens_generated": 492, + "tok_per_sec": 253.480906586 + }, + "bs8": { + "bs": 8, + "elapsed_s": 1.9738178760162555, + "tokens_generated": 984, + "tok_per_sec": 498.5262378847238 + }, + "bs32": { + "bs": 32, + "elapsed_s": 2.400167693005642, + "tokens_generated": 3936, + "tok_per_sec": 1639.8854177855762 + } + } +} \ No newline at end of file diff --git a/tokenizer_fertility.json b/tokenizer_fertility.json new file mode 100644 index 0000000..9cd267f --- /dev/null +++ b/tokenizer_fertility.json @@ -0,0 +1,10 @@ +{ + "words": 368, + "ours_tokens": 664, + "ours_fertility": 1.8043478260869565, + "baseline_tokens": 803, + "baseline_fertility": 2.182065217391304, + "reduction_pct": 17.310087173100865, + "ours_vocab_size": 178697, + "baseline_vocab_size": 151665 +} \ No newline at end of file