初始化项目，由ModelHub XC社区提供模型
Model: belal212/Fattah-2.5B-preview Source: Original Platform
This commit is contained in:
ModelHub XC
36
.gitattributes
vendored
Normal file
36
.gitattributes
vendored
Normal file
@@ -0,0 +1,36 @@
|
|||||||
|
*.7z filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.arrow filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.bin filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.ftz filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.gz filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.h5 filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.joblib filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.model filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.npy filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.npz filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.onnx filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.ot filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.parquet filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.pb filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.pickle filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.pkl filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.pt filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.pth filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.rar filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
||||||
|
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.tar filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.tflite filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.tgz filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.wasm filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.xz filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.zip filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.zst filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
||||||
|
tokenizer.json filter=lfs diff=lfs merge=lfs -text
|
||||||
410
README.md
Normal file
410
README.md
Normal file
@@ -0,0 +1,410 @@
|
|||||||
|
---
|
||||||
|
language:
|
||||||
|
- ar
|
||||||
|
- en
|
||||||
|
license: apache-2.0
|
||||||
|
tags:
|
||||||
|
- egyptian-arabic
|
||||||
|
- arabic
|
||||||
|
- causal-lm
|
||||||
|
- continual-pretraining
|
||||||
|
- instruction-tuning
|
||||||
|
- dialect
|
||||||
|
- qwen3
|
||||||
|
pipeline_tag: text-generation
|
||||||
|
base_model: Qwen/Qwen3-1.7B-Base
|
||||||
|
datasets:
|
||||||
|
- MBZUAI-Paris/Egyptian-SFT-Mixture
|
||||||
|
- UBC-NLP/nilechat-fw-edu-egy
|
||||||
|
model-index:
|
||||||
|
- name: Fattah-2.5B
|
||||||
|
results:
|
||||||
|
- task:
|
||||||
|
type: text-generation
|
||||||
|
dataset:
|
||||||
|
name: EgyptianMMLU
|
||||||
|
type: MBZUAI-Paris/EgyptianMMLU
|
||||||
|
metrics:
|
||||||
|
- type: accuracy
|
||||||
|
value: 38.4
|
||||||
|
name: EgyptianMMLU (acc)
|
||||||
|
- task:
|
||||||
|
type: text-generation
|
||||||
|
dataset:
|
||||||
|
name: EgyptianPIQA
|
||||||
|
type: MBZUAI-Paris/EgyptianPIQA
|
||||||
|
metrics:
|
||||||
|
- type: accuracy
|
||||||
|
value: 61.3
|
||||||
|
name: EgyptianPIQA (acc)
|
||||||
|
- task:
|
||||||
|
type: text-generation
|
||||||
|
dataset:
|
||||||
|
name: Belebele-Arz
|
||||||
|
type: facebook/belebele
|
||||||
|
metrics:
|
||||||
|
- type: accuracy
|
||||||
|
value: 40.78
|
||||||
|
name: Belebele-Arz (acc)
|
||||||
|
- task:
|
||||||
|
type: text-generation
|
||||||
|
dataset:
|
||||||
|
name: EgyptianHellaSwag
|
||||||
|
type: MBZUAI-Paris/EgyptianHellaSwag
|
||||||
|
metrics:
|
||||||
|
- type: accuracy
|
||||||
|
value: 24.0
|
||||||
|
name: EgyptianHellaSwag (acc_norm)
|
||||||
|
- task:
|
||||||
|
type: text-generation
|
||||||
|
dataset:
|
||||||
|
name: EgyptianWinoGrande
|
||||||
|
type: MBZUAI-Paris/EgyptianWinoGrande
|
||||||
|
metrics:
|
||||||
|
- type: accuracy
|
||||||
|
value: 49.4
|
||||||
|
name: EgyptianWinoGrande (acc)
|
||||||
|
- task:
|
||||||
|
type: text-generation
|
||||||
|
dataset:
|
||||||
|
name: EgyptianOpenBookQA
|
||||||
|
type: MBZUAI-Paris/EgyptianOpenBookQA
|
||||||
|
metrics:
|
||||||
|
- type: accuracy
|
||||||
|
value: 27.96
|
||||||
|
name: EgyptianOpenBookQA (acc)
|
||||||
|
---
|
||||||
|
|
||||||
|
<div align="center">
|
||||||
|
|
||||||
|
# فتاح — Fattah-2.5B
|
||||||
|
|
||||||
|
### نموذج لغوي مصري مبني على Qwen3 بتقنية Depth-Up Scaling
|
||||||
|
|
||||||
|
**Egyptian Arabic LLM Built on Qwen3 with Depth-Up Scaling**
|
||||||
|
|
||||||
|
[](https://opensource.org/licenses/Apache-2.0)
|
||||||
|
[]()
|
||||||
|
[]()
|
||||||
|
[]()
|
||||||
|
|
||||||
|
</div>
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Overview
|
||||||
|
|
||||||
|
**Fattah** (فتاح — meaning "the opener" or "the one who opens doors") is a 2.5B parameter Large Language Model specialized for **Egyptian Arabic**, the most widely spoken Arabic dialect with over 100 million native speakers.
|
||||||
|
|
||||||
|
Fattah is built through a novel three-stage pipeline:
|
||||||
|
1. **Depth-Up Scaling (DUS)** — expanding Qwen3-1.7B from 28 to 40 transformer layers
|
||||||
|
2. **Continual Pre-Training (CPT)** — trained on a ~8.59B token Egyptian Arabic corpus, processing 5.51B tokens (64.1% of the full dataset)
|
||||||
|
3. **Supervised Fine-Tuning (SFT)** — 400K Egyptian Arabic instruction-response pairs
|
||||||
|
|
||||||
|
> ⚠️ **Note:** This is the **pre-DPO** version (CPT + SFT only). A DPO-aligned version (`Fattah-2.5B-v2`) is coming soon with improved factual accuracy, reduced hallucination, and better instruction following.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Model Details
|
||||||
|
|
||||||
|
| Property | Value |
|
||||||
|
|---|---|
|
||||||
|
| **Model Name** | Fattah-2.5B |
|
||||||
|
| **Base Model** | Qwen/Qwen3-1.7B-Base |
|
||||||
|
| **Architecture** | Qwen3 (expanded via DUS) |
|
||||||
|
| **Parameters** | 2,635,771,904 (~2.64B) |
|
||||||
|
| **Transformer Layers** | 40 (expanded from 28) |
|
||||||
|
| **Hidden Size** | 2048 |
|
||||||
|
| **Context Length** | 64K tokens (YaRN extended) |
|
||||||
|
| **Language** | Egyptian Arabic (primary), MSA, English |
|
||||||
|
| **License** | Apache 2.0 |
|
||||||
|
| **Training Compute** | 2× NVIDIA A6000 48GB |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Training Pipeline
|
||||||
|
|
||||||
|
### Stage 1 — Depth-Up Scaling (DUS)
|
||||||
|
|
||||||
|
Starting from `Qwen/Qwen3-1.7B-Base`, we applied **Depth-Up Scaling** surgery — the same technique used in SOLAR-10.7B — to expand the model from 28 to 40 transformer layers, increasing parameter count from 1.7B to ~2.5B without any training.
|
||||||
|
|
||||||
|
```
|
||||||
|
Qwen3-1.7B-Base (28 layers)
|
||||||
|
↓ DUS Surgery
|
||||||
|
Fattah-DUS (40 layers, ~2.5B)
|
||||||
|
```
|
||||||
|
|
||||||
|
Layer expansion strategy: concatenate layers `[0-23]` + layers `[4-27]`, creating a deeper model that inherits the base model's knowledge while providing additional capacity for Egyptian Arabic adaptation.
|
||||||
|
|
||||||
|
### Stage 2 — Continual Pre-Training (CPT)
|
||||||
|
|
||||||
|
| Parameter | Value |
|
||||||
|
|---|---|
|
||||||
|
| Dataset | Custom Egyptian Arabic corpus (~8.59B tokens) |
|
||||||
|
| Total dataset tokens | **~8.59B tokens** |
|
||||||
|
| Tokens processed | **5.51B tokens** (64.1% of dataset) |
|
||||||
|
| Training steps | 42,000 |
|
||||||
|
| Learning rate | 1e-5 (cosine decay) |
|
||||||
|
| Sequence length | 4096 |
|
||||||
|
| Batch size | 2 per GPU × 8 grad accum × 2 GPUs = 131,072 tokens/step |
|
||||||
|
| Framework | ms-swift + DeepSpeed ZeRO-1 |
|
||||||
|
| Final loss | 1.824 |
|
||||||
|
|
||||||
|
**Dataset composition:**
|
||||||
|
- 51.7% Egyptian Arabic (web, subtitles, social media, educational)
|
||||||
|
- 22.1% Modern Standard Arabic (MSA)
|
||||||
|
- 13.8% English
|
||||||
|
- 12.4% Code
|
||||||
|
|
||||||
|
### Stage 3 — Supervised Fine-Tuning (SFT)
|
||||||
|
|
||||||
|
| Parameter | Value |
|
||||||
|
|---|---|
|
||||||
|
| Dataset | `MBZUAI-Paris/Egyptian-SFT-Mixture` (400K samples) |
|
||||||
|
| Epochs | 2 |
|
||||||
|
| Learning rate | 5e-6 (cosine decay) |
|
||||||
|
| Final eval loss | **1.668** |
|
||||||
|
| Final token accuracy | **67.01%** |
|
||||||
|
| Training time | ~19 hours |
|
||||||
|
|
||||||
|
### Context Extension — YaRN
|
||||||
|
|
||||||
|
After SFT, the context window was extended from 32K to **64K tokens** using YaRN (Yet another RoPE extensioN):
|
||||||
|
|
||||||
|
```json
|
||||||
|
"rope_scaling": {
|
||||||
|
"rope_type": "yarn",
|
||||||
|
"factor": 2.0,
|
||||||
|
"original_max_position_embeddings": 32768
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Evaluation Results
|
||||||
|
|
||||||
|
All evaluations use **zero-shot log-likelihood scoring** (same methodology as NileChat paper). HellaSwag uses length-normalized accuracy (`acc_norm`); all other benchmarks use unnormalized accuracy (`acc`).
|
||||||
|
|
||||||
|
### Arabic Script Benchmarks — Full Comparison
|
||||||
|
|
||||||
|
|
||||||
|
All evaluations use **zero-shot log-likelihood scoring**. HellaSwag uses `acc_norm` (length-normalized accuracy). All other benchmarks use `acc` (unnormalized accuracy).
|
||||||
|
|
||||||
|
Published baselines are from the **NileChat paper (Table 1)**. Fattah rows use our custom evaluation harness with identical zero-shot methodology.
|
||||||
|
|
||||||
|
## Arabic Script Benchmarks
|
||||||
|
|
||||||
|
| Model | Params | MMLU | Belebele | HellaSwag† | PIQA | WinoGrande | OpenBookQA | **Avg** |
|
||||||
|
|---|---|---:|---:|---:|---:|---:|---:|---:|
|
||||||
|
| Nile-Chat-12B | 12B | 62.59 | 70.69 | 64.04 | 63.53 | 42.06 | 53.13 | **59.34** |
|
||||||
|
| gemma-3-12b-it | 12B | 61.55 | 77.00 | 49.49 | 63.53 | 38.03 | 48.86 | **56.41** |
|
||||||
|
| Qwen2.5-14B-Instruct | 14B | 60.81 | 72.33 | 55.84 | 59.97 | 38.26 | 50.28 | **56.25** |
|
||||||
|
| Nile-Chat-3x4B-A6B | MoE | 52.13 | 75.44 | 59.30 | 57.91 | 41.16 | 48.39 | **55.72** |
|
||||||
|
| Nile-Chat-2x4B-A6B | MoE | 52.05 | 73.89 | 59.69 | 62.26 | 41.61 | 44.07 | **55.60** |
|
||||||
|
| AceGPT-v2-8b-chat | 8B | 55.25 | 73.33 | 53.14 | 58.39 | 39.82 | 47.16 | **54.52** |
|
||||||
|
| Nile-Chat-4B | 4B | 50.25 | 68.56 | 55.92 | 61.87 | 40.94 | 46.02 | **53.93** |
|
||||||
|
| c4ai-command-r7b | 7B | 70.67 | 61.84 | 50.39 | 57.20 | 36.91 | 46.02 | **53.84** |
|
||||||
|
| ALLaM-7B-Instruct | 7B | 67.67 | 66.10 | 57.29 | 62.18 | 40.04 | 67.10 | **60.06** |
|
||||||
|
| gemma-2-9b-it | 9B | 49.44 | 61.35 | 49.53 | 61.79 | 35.79 | 48.01 | **50.99** |
|
||||||
|
| jais-adapted-13b-chat | 13B | 50.03 | 65.33 | 47.53 | 56.72 | 37.14 | 41.76 | **49.75** |
|
||||||
|
| jais-family-13b-chat | 13B | 44.85 | 66.33 | 52.99 | 57.91 | 36.91 | 38.64 | **49.61** |
|
||||||
|
| jais-family-6p7b-chat | 7B | 42.60 | 57.33 | 49.18 | 62.23 | 33.33 | 37.50 | **47.03** |
|
||||||
|
| gemma-3-4b-it | 4B | 38.56 | 60.32 | 42.56 | 56.49 | 35.79 | 46.73 | **46.74** |
|
||||||
|
| Qwen2.5-7B-Instruct | 7B | 64.22 | 58.02 | 45.47 | 56.41 | 38.70 | 11.34 | **45.69** |
|
||||||
|
| jais-adapted-7b-chat | 7B | 40.96 | 55.67 | 40.85 | 56.50 | 32.89 | 42.33 | **44.87** |
|
||||||
|
| Llama-3.1-8B-Instruct | 8B | 55.89 | 57.97 | 43.10 | 54.27 | 35.57 | 9.06 | **42.64** |
|
||||||
|
| | | | | | | | | |
|
||||||
|
| **Fattah-2.5B (post-SFT)** ⭐ | **2.5B** | **38.40** | **40.78** | **24.00** | **61.30** | **49.40** | **27.96** | **40.31** |
|
||||||
|
† HellaSwag uses `acc_norm` (length-normalized accuracy). All other benchmarks use `acc`.
|
||||||
|
‡ Published baselines are from the NileChat paper (Table 1) — these are instruction-tuned + RLHF-aligned models.
|
||||||
|
⭐ Best Fattah checkpoint (pre-DPO).
|
||||||
|
|
||||||
|
## Key Highlights
|
||||||
|
|
||||||
|
- **PIQA (61.3%)** — Fattah outperforms Qwen2.5-7B (56.4%), gemma-3-4b (56.5%), Llama-3.1-8B (54.3%), and all jais models despite being 2.5B
|
||||||
|
- **WinoGrande (49.4%)** — Fattah scores higher than every published baseline in the table, including models 3–5× larger
|
||||||
|
- **Average gap** — Fattah post-SFT (40.31%) is behind Nile-Chat-4B (53.93%) by 13.6 points; DPO alignment is expected to close this gap significantly
|
||||||
|
- **Comparable baselines** — most fair comparison is with gemma-3-4b-it (4B, 46.74%) — Fattah is 2.5B and pre-DPO, 6.4 points behind a fully aligned 4B model
|
||||||
|
|
||||||
|
### Full Training Journey (Base → DUS → CPT → SFT)
|
||||||
|
|
||||||
|
| Benchmark | Base 1.7B | DUS 2.5B | Post-CPT | **Post-SFT** | Net (Base→SFT) |
|
||||||
|
|---|---|---|---|---|---|
|
||||||
|
| EgyptianMMLU | 34.07% | 29.20% | 37.07% | **38.40%** | **+4.33%** ✅ |
|
||||||
|
| EgyptianPIQA | 54.80% | 51.90% | 61.10% | **61.30%** | **+6.50%** ✅ |
|
||||||
|
| Belebele-Arz | 37.00% | 32.78% | 41.56% | **40.78%** | **+3.78%** ✅ |
|
||||||
|
| EgyHellaSwag | 25.00% | 23.60% | 21.40% | **24.00%** | **−1.00%** ⚠️ |
|
||||||
|
| WinoGrande | 49.40% | 49.40% | 49.40% | **49.40%** | **0.00%** ➡️ |
|
||||||
|
| OpenBookQA | 21.03% | 17.67% | 27.74% | **27.96%** | **+6.93%** ✅ |
|
||||||
|
| **Average** | **36.88%** | **34.09%** | **39.71%** | **40.31%** | **+3.43%** ✅ |
|
||||||
|
| EGY Perplexity | 18.84 | 46.31 | **6.69** | — | **−12.15** ✅ |
|
||||||
|
|
||||||
|
Key observations:
|
||||||
|
- DUS surgery caused an expected temporary regression (34.09%) as the new layers were randomly initialized
|
||||||
|
- CPT recovered and surpassed the base (39.71%), acquiring strong Egyptian Arabic dialect knowledge
|
||||||
|
- SFT further improved average to **40.31%**, with MMLU +1.33% and HellaSwag recovering from 21.4% → 24.0%
|
||||||
|
- EGY Perplexity improvement of ×2.8 (18.84 → 6.69) confirms deep dialect acquisition during CPT
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Usage
|
||||||
|
|
||||||
|
### Installation
|
||||||
|
|
||||||
|
```bash
|
||||||
|
pip install transformers>=4.51.0 torch accelerate
|
||||||
|
```
|
||||||
|
|
||||||
|
### Basic Chat
|
||||||
|
|
||||||
|
```python
|
||||||
|
from transformers import AutoModelForCausalLM, AutoTokenizer
|
||||||
|
import torch
|
||||||
|
|
||||||
|
model_name = "belal212/Fattah-2.5B-preview"
|
||||||
|
|
||||||
|
tokenizer = AutoTokenizer.from_pretrained(model_name)
|
||||||
|
model = AutoModelForCausalLM.from_pretrained(
|
||||||
|
model_name,
|
||||||
|
torch_dtype=torch.bfloat16,
|
||||||
|
device_map="auto"
|
||||||
|
)
|
||||||
|
|
||||||
|
messages = [
|
||||||
|
{
|
||||||
|
"role": "system",
|
||||||
|
"content": "أنت فتاح، مساعد ذكي ومفيد بتتكلم العربي المصري."
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"role": "user",
|
||||||
|
"content": "كلمني عن القاهرة"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
|
||||||
|
text = tokenizer.apply_chat_template(
|
||||||
|
messages,
|
||||||
|
tokenize=False,
|
||||||
|
add_generation_prompt=True,
|
||||||
|
enable_thinking=False # disable thinking mode for conversational use
|
||||||
|
)
|
||||||
|
|
||||||
|
inputs = tokenizer(text, return_tensors="pt").to(model.device)
|
||||||
|
|
||||||
|
with torch.no_grad():
|
||||||
|
outputs = model.generate(
|
||||||
|
**inputs,
|
||||||
|
max_new_tokens=512,
|
||||||
|
do_sample=True,
|
||||||
|
temperature=0.7,
|
||||||
|
top_p=0.9,
|
||||||
|
repetition_penalty=1.1,
|
||||||
|
pad_token_id=tokenizer.eos_token_id
|
||||||
|
)
|
||||||
|
|
||||||
|
response = tokenizer.decode(
|
||||||
|
outputs[0][inputs["input_ids"].shape[1]:],
|
||||||
|
skip_special_tokens=True
|
||||||
|
)
|
||||||
|
print(response)
|
||||||
|
```
|
||||||
|
|
||||||
|
### With Thinking Mode (for complex reasoning)
|
||||||
|
|
||||||
|
```python
|
||||||
|
messages = [
|
||||||
|
{
|
||||||
|
"role": "system",
|
||||||
|
"content": "أنت فتاح، مساعد ذكي بتفكر خطوة بخطوة قبل ما تجاوب."
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"role": "user",
|
||||||
|
"content": "ازاي أحسن خوارزمية للـ sorting في Python؟"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
|
||||||
|
text = tokenizer.apply_chat_template(
|
||||||
|
messages,
|
||||||
|
tokenize=False,
|
||||||
|
add_generation_prompt=True,
|
||||||
|
enable_thinking=True # activate <think> mode
|
||||||
|
)
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Intended Use
|
||||||
|
|
||||||
|
Fattah is designed for:
|
||||||
|
|
||||||
|
- ✅ Egyptian Arabic conversational AI
|
||||||
|
- ✅ Question answering in Egyptian dialect
|
||||||
|
- ✅ Text generation and creative writing in Egyptian Arabic
|
||||||
|
- ✅ RAG-based knowledge retrieval systems
|
||||||
|
- ✅ Foundation for Fattah-Coding (Python + React/TS specialist — coming soon)
|
||||||
|
- ✅ Agent systems requiring Egyptian Arabic understanding
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Limitations
|
||||||
|
|
||||||
|
- **Factual hallucination**: As a 2.5B model without DPO alignment, Fattah may confidently generate incorrect facts. A DPO-aligned version is in development.
|
||||||
|
- **Knowledge cutoff**: Training data has a knowledge cutoff. Recent events are not known.
|
||||||
|
- **Dialect coverage**: Optimized for Egyptian Arabic. Performance on other Arabic dialects is not guaranteed.
|
||||||
|
- **Model size**: At 2.5B parameters, Fattah cannot match the factual depth of larger models. Use RAG for knowledge-intensive applications.
|
||||||
|
- **Pre-DPO**: This version has not undergone preference optimization. Responses may occasionally be over-cautious or inconsistent in style.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Roadmap
|
||||||
|
|
||||||
|
| Version | Status | Description |
|
||||||
|
|---|---|---|
|
||||||
|
| Fattah-2.5B | ✅ Released | CPT + SFT, Egyptian Arabic assistant |
|
||||||
|
| Fattah-2.5B-v2 | 🔄 In progress | + DPO alignment (Egyptian-DPO-Mixture) |
|
||||||
|
| Fattah-Python-2.5B | ⏳ Planned | Fattah + Python/AI coding specialization |
|
||||||
|
| Fattah-React-2.5B | ⏳ Planned | Fattah + React/TypeScript specialization |
|
||||||
|
| Fattah-Coding-MoE | ⏳ Planned | MoE with LLM-gated routing between Python + React experts |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Training Infrastructure
|
||||||
|
|
||||||
|
- **GPUs**: 2× NVIDIA A6000 48GB
|
||||||
|
- **Framework**: [ms-swift](https://github.com/modelscope/ms-swift) 4.0.2
|
||||||
|
- **Distributed**: DeepSpeed ZeRO Stage 1
|
||||||
|
- **Attention**: Flash Attention 2.3.6
|
||||||
|
- **Mixed precision**: bfloat16
|
||||||
|
- **Total compute**: ~60 GPU-hours (CPT) + ~19 GPU-hours (SFT)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Citation
|
||||||
|
|
||||||
|
If you use Fattah in your research, please cite:
|
||||||
|
|
||||||
|
```bibtex
|
||||||
|
@misc{fattah2026,
|
||||||
|
title = {Fattah: Egyptian Arabic LLM via Depth-Up Scaling and Continual Pre-Training},
|
||||||
|
author = {Belal},
|
||||||
|
year = {2026},
|
||||||
|
publisher = {HuggingFace},
|
||||||
|
howpublished = {\url{https://huggingface.co/belal212/Fattah-2.5B-preview}},
|
||||||
|
note = {Pre-DPO version}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Acknowledgements
|
||||||
|
|
||||||
|
- [Qwen Team](https://huggingface.co/Qwen) for the Qwen3-1.7B-Base model
|
||||||
|
- [MBZUAI-Paris](https://huggingface.co/MBZUAI-Paris) for the Egyptian-SFT-Mixture dataset and NileChat benchmarks
|
||||||
|
- [UBC-NLP](https://huggingface.co/UBC-NLP) for the NileChat pre-training corpus
|
||||||
|
- [ms-swift](https://github.com/modelscope/ms-swift) for the training framework
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
<div align="center">
|
||||||
|
<b>فتاح — بيفتح أبواب الذكاء الاصطناعي للعربي المصري</b><br>
|
||||||
|
<i>Fattah — Opening the doors of AI for Egyptian Arabic speakers</i>
|
||||||
|
</div>
|
||||||
85
chat_template.jinja
Normal file
85
chat_template.jinja
Normal file
@@ -0,0 +1,85 @@
|
|||||||
|
{%- if tools %}
|
||||||
|
{{- '<|im_start|>system\n' }}
|
||||||
|
{%- if messages[0].role == 'system' %}
|
||||||
|
{{- messages[0].content + '\n\n' }}
|
||||||
|
{%- endif %}
|
||||||
|
{{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
|
||||||
|
{%- for tool in tools %}
|
||||||
|
{{- "\n" }}
|
||||||
|
{{- tool | tojson }}
|
||||||
|
{%- endfor %}
|
||||||
|
{{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
|
||||||
|
{%- else %}
|
||||||
|
{%- if messages[0].role == 'system' %}
|
||||||
|
{{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
|
||||||
|
{%- endif %}
|
||||||
|
{%- endif %}
|
||||||
|
{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
|
||||||
|
{%- for message in messages[::-1] %}
|
||||||
|
{%- set index = (messages|length - 1) - loop.index0 %}
|
||||||
|
{%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('<tool_response>') and message.content.endswith('</tool_response>')) %}
|
||||||
|
{%- set ns.multi_step_tool = false %}
|
||||||
|
{%- set ns.last_query_index = index %}
|
||||||
|
{%- endif %}
|
||||||
|
{%- endfor %}
|
||||||
|
{%- for message in messages %}
|
||||||
|
{%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
|
||||||
|
{{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
|
||||||
|
{%- elif message.role == "assistant" %}
|
||||||
|
{%- set content = message.content %}
|
||||||
|
{%- set reasoning_content = '' %}
|
||||||
|
{%- if message.reasoning_content is defined and message.reasoning_content is not none %}
|
||||||
|
{%- set reasoning_content = message.reasoning_content %}
|
||||||
|
{%- else %}
|
||||||
|
{%- if '</think>' in message.content %}
|
||||||
|
{%- set content = message.content.split('</think>')[-1].lstrip('\n') %}
|
||||||
|
{%- set reasoning_content = message.content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
|
||||||
|
{%- endif %}
|
||||||
|
{%- endif %}
|
||||||
|
{%- if loop.index0 > ns.last_query_index %}
|
||||||
|
{%- if loop.last or (not loop.last and reasoning_content) %}
|
||||||
|
{{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content.strip('\n') + '\n</think>\n\n' + content.lstrip('\n') }}
|
||||||
|
{%- else %}
|
||||||
|
{{- '<|im_start|>' + message.role + '\n' + content }}
|
||||||
|
{%- endif %}
|
||||||
|
{%- else %}
|
||||||
|
{{- '<|im_start|>' + message.role + '\n' + content }}
|
||||||
|
{%- endif %}
|
||||||
|
{%- if message.tool_calls %}
|
||||||
|
{%- for tool_call in message.tool_calls %}
|
||||||
|
{%- if (loop.first and content) or (not loop.first) %}
|
||||||
|
{{- '\n' }}
|
||||||
|
{%- endif %}
|
||||||
|
{%- if tool_call.function %}
|
||||||
|
{%- set tool_call = tool_call.function %}
|
||||||
|
{%- endif %}
|
||||||
|
{{- '<tool_call>\n{"name": "' }}
|
||||||
|
{{- tool_call.name }}
|
||||||
|
{{- '", "arguments": ' }}
|
||||||
|
{%- if tool_call.arguments is string %}
|
||||||
|
{{- tool_call.arguments }}
|
||||||
|
{%- else %}
|
||||||
|
{{- tool_call.arguments | tojson }}
|
||||||
|
{%- endif %}
|
||||||
|
{{- '}\n</tool_call>' }}
|
||||||
|
{%- endfor %}
|
||||||
|
{%- endif %}
|
||||||
|
{{- '<|im_end|>\n' }}
|
||||||
|
{%- elif message.role == "tool" %}
|
||||||
|
{%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
|
||||||
|
{{- '<|im_start|>user' }}
|
||||||
|
{%- endif %}
|
||||||
|
{{- '\n<tool_response>\n' }}
|
||||||
|
{{- message.content }}
|
||||||
|
{{- '\n</tool_response>' }}
|
||||||
|
{%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
|
||||||
|
{{- '<|im_end|>\n' }}
|
||||||
|
{%- endif %}
|
||||||
|
{%- endif %}
|
||||||
|
{%- endfor %}
|
||||||
|
{%- if add_generation_prompt %}
|
||||||
|
{{- '<|im_start|>assistant\n' }}
|
||||||
|
{%- if enable_thinking is defined and enable_thinking is false %}
|
||||||
|
{{- '<think>\n\n</think>\n\n' }}
|
||||||
|
{%- endif %}
|
||||||
|
{%- endif %}
|
||||||
80
config.json
Normal file
80
config.json
Normal file
@@ -0,0 +1,80 @@
|
|||||||
|
{
|
||||||
|
"architectures": [
|
||||||
|
"Qwen3ForCausalLM"
|
||||||
|
],
|
||||||
|
"attention_bias": false,
|
||||||
|
"attention_dropout": 0.0,
|
||||||
|
"bos_token_id": null,
|
||||||
|
"dtype": "bfloat16",
|
||||||
|
"eos_token_id": 151643,
|
||||||
|
"head_dim": 128,
|
||||||
|
"hidden_act": "silu",
|
||||||
|
"hidden_size": 2048,
|
||||||
|
"initializer_range": 0.02,
|
||||||
|
"intermediate_size": 6144,
|
||||||
|
"layer_types": [
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention"
|
||||||
|
],
|
||||||
|
"max_position_embeddings": 65536,
|
||||||
|
"max_window_layers": 28,
|
||||||
|
"model_type": "qwen3",
|
||||||
|
"num_attention_heads": 16,
|
||||||
|
"num_hidden_layers": 40,
|
||||||
|
"num_key_value_heads": 8,
|
||||||
|
"pad_token_id": 151643,
|
||||||
|
"rms_norm_eps": 1e-06,
|
||||||
|
"rope_parameters": {
|
||||||
|
"rope_theta": 1000000,
|
||||||
|
"rope_type": "default"
|
||||||
|
},
|
||||||
|
"sliding_window": null,
|
||||||
|
"tie_word_embeddings": false,
|
||||||
|
"transformers_version": "5.3.0",
|
||||||
|
"use_cache": false,
|
||||||
|
"use_sliding_window": false,
|
||||||
|
"vocab_size": 151936,
|
||||||
|
"rope_scaling": {
|
||||||
|
"rope_type": "yarn",
|
||||||
|
"factor": 2.0,
|
||||||
|
"original_max_position_embeddings": 32768
|
||||||
|
}
|
||||||
|
}
|
||||||
12
generation_config.json
Normal file
12
generation_config.json
Normal file
@@ -0,0 +1,12 @@
|
|||||||
|
{
|
||||||
|
"_from_model_config": true,
|
||||||
|
"bos_token_id": 151643,
|
||||||
|
"eos_token_id": [
|
||||||
|
151643,
|
||||||
|
151645
|
||||||
|
],
|
||||||
|
"output_attentions": false,
|
||||||
|
"output_hidden_states": false,
|
||||||
|
"transformers_version": "5.3.0",
|
||||||
|
"use_cache": true
|
||||||
|
}
|
||||||
3
model.safetensors
Normal file
3
model.safetensors
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:00c8479a45be2bbc0926587b7687dcb8480249fa541eab72268f82272df13b2f
|
||||||
|
size 5271594904
|
||||||
3
tokenizer.json
Normal file
3
tokenizer.json
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506
|
||||||
|
size 11422650
|
||||||
29
tokenizer_config.json
Normal file
29
tokenizer_config.json
Normal file
@@ -0,0 +1,29 @@
|
|||||||
|
{
|
||||||
|
"add_prefix_space": false,
|
||||||
|
"backend": "tokenizers",
|
||||||
|
"bos_token": null,
|
||||||
|
"clean_up_tokenization_spaces": false,
|
||||||
|
"eos_token": "<|endoftext|>",
|
||||||
|
"errors": "replace",
|
||||||
|
"extra_special_tokens": [
|
||||||
|
"<|im_start|>",
|
||||||
|
"<|im_end|>",
|
||||||
|
"<|object_ref_start|>",
|
||||||
|
"<|object_ref_end|>",
|
||||||
|
"<|box_start|>",
|
||||||
|
"<|box_end|>",
|
||||||
|
"<|quad_start|>",
|
||||||
|
"<|quad_end|>",
|
||||||
|
"<|vision_start|>",
|
||||||
|
"<|vision_end|>",
|
||||||
|
"<|vision_pad|>",
|
||||||
|
"<|image_pad|>",
|
||||||
|
"<|video_pad|>"
|
||||||
|
],
|
||||||
|
"is_local": true,
|
||||||
|
"model_max_length": 131072,
|
||||||
|
"pad_token": "<|endoftext|>",
|
||||||
|
"split_special_tokens": false,
|
||||||
|
"tokenizer_class": "Qwen2Tokenizer",
|
||||||
|
"unk_token": null
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user