初始化项目，由ModelHub XC社区提供模型
Model: Chamaka8/Serendip-LLM-CPT-SFT-v2 Source: Original Platform
This commit is contained in:
ModelHub XC
41
training_scripts/README.md
Normal file
41
training_scripts/README.md
Normal file
@@ -0,0 +1,41 @@
|
||||
# Training Scripts
|
||||
|
||||
This folder contains all training scripts used in the SerendipLLM V2 project.
|
||||
|
||||
## ✅ Final Script Used (ACTUAL TRAINING)
|
||||
|
||||
**`train_v2_fast.py`** ← This is the exact script that trained the final model!
|
||||
|
||||
- Training time: 26.5 hours
|
||||
- Loss: 0.54 → 0.27 (50% improvement)
|
||||
- Epochs: 3
|
||||
- Batch size: 8 (effective 32)
|
||||
- LoRA rank: 64
|
||||
- Dataset: 309,328 examples
|
||||
|
||||
## 📝 Other Scripts (Development/Testing)
|
||||
|
||||
- `train_phase1_fixed.py` - Initial attempt (slower, 512 tokens)
|
||||
- `continue_training.py` - Script for resuming training (not used)
|
||||
|
||||
## 🎯 To Reproduce
|
||||
|
||||
Use `train_v2_fast.py` with these settings:
|
||||
- GPU: A100 80GB
|
||||
- Dataset: Chamaka8/Serendip-sft-sinhala (serendipllm_sft_final_train_v2.json)
|
||||
- Time: ~27 hours
|
||||
- Cost: ~$37
|
||||
|
||||
## 📊 Training Results
|
||||
```
|
||||
Epoch 1: Loss 0.28
|
||||
Epoch 2: Loss 0.24
|
||||
Epoch 3: Loss 0.27
|
||||
Final average loss: 0.27
|
||||
```
|
||||
|
||||
## 🔗 Related Resources
|
||||
|
||||
- Model: https://huggingface.co/Chamaka8/Serendip-LLM-CPT-SFT-v2
|
||||
- Dataset: https://huggingface.co/datasets/Chamaka8/Serendip-sft-sinhala
|
||||
- Base CPT: https://huggingface.co/Chamaka8/serendib-llm-cpt-llama3-8b
|
||||
52
training_scripts/TRAINING_LOG.md
Normal file
52
training_scripts/TRAINING_LOG.md
Normal file
@@ -0,0 +1,52 @@
|
||||
# Training Log Summary
|
||||
|
||||
## Final Training Run (train_v2_fast.py)
|
||||
|
||||
**Start:** February 18, 2026, ~17:30
|
||||
**End:** February 19, 2026, ~20:00
|
||||
**Duration:** 26.5 hours
|
||||
|
||||
### Loss Progression
|
||||
|
||||
| Epoch | Loss |
|
||||
|-------|------|
|
||||
| 0.95 | 0.28 |
|
||||
| 1.90 | 0.24 |
|
||||
| 3.00 | 0.27 |
|
||||
|
||||
**Final training loss:** 0.27
|
||||
|
||||
### Configuration Used
|
||||
```python
|
||||
num_train_epochs = 3
|
||||
per_device_train_batch_size = 8
|
||||
gradient_accumulation_steps = 4 # Effective batch = 32
|
||||
learning_rate = 2e-5
|
||||
max_length = 384 # tokens
|
||||
warmup_steps = 200
|
||||
weight_decay = 0.01
|
||||
|
||||
# LoRA Config
|
||||
lora_r = 64
|
||||
lora_alpha = 128
|
||||
lora_target_modules = [
|
||||
"q_proj", "k_proj", "v_proj",
|
||||
"o_proj", "gate_proj", "up_proj"
|
||||
]
|
||||
lora_dropout = 0.05
|
||||
```
|
||||
|
||||
### Hardware
|
||||
|
||||
- GPU: NVIDIA A100 SXM 80GB
|
||||
- Training framework: Transformers + PEFT
|
||||
- Mixed precision: FP16
|
||||
|
||||
### Dataset
|
||||
|
||||
- Source: Chamaka8/Serendip-sft-sinhala
|
||||
- File: serendipllm_sft_final_train_v2.json
|
||||
- Examples: 309,328
|
||||
- News classification: 45,080 examples
|
||||
- General Sinhala: 205,403 examples
|
||||
- QA pairs: 29,390 examples
|
||||
129
training_scripts/archive/train_phase1_fixed.py
Normal file
129
training_scripts/archive/train_phase1_fixed.py
Normal file
@@ -0,0 +1,129 @@
|
||||
import torch, os, gc
|
||||
from transformers import AutoModelForCausalLM, AutoTokenizer, TrainingArguments, Trainer, DataCollatorForLanguageModeling
|
||||
from datasets import load_dataset
|
||||
from peft import LoraConfig, get_peft_model
|
||||
|
||||
os.environ["TOKENIZERS_PARALLELISM"] = "false"
|
||||
|
||||
print("="*70)
|
||||
print("SERENDIPLLM V2 - FRESH TRAINING WITH FIXED DATASET")
|
||||
print("="*70)
|
||||
print("News data: 45,080 examples (was 3!)")
|
||||
print("Total: 309,328 examples")
|
||||
print("Epochs: 3")
|
||||
print("="*70)
|
||||
|
||||
BASE_MODEL = "Chamaka8/serendib-llm-cpt-llama3-8b"
|
||||
OUTPUT_DIR = "./SerendipLLM-V2"
|
||||
FINAL_MODEL = "Chamaka8/Serendip-LLM-CPT-SFT-v2"
|
||||
|
||||
gc.collect()
|
||||
torch.cuda.empty_cache()
|
||||
|
||||
print("\nLoading tokenizer...")
|
||||
tokenizer = AutoTokenizer.from_pretrained(BASE_MODEL, use_fast=True)
|
||||
tokenizer.pad_token = tokenizer.eos_token
|
||||
|
||||
print("Loading model...")
|
||||
model = AutoModelForCausalLM.from_pretrained(
|
||||
BASE_MODEL,
|
||||
torch_dtype=torch.float16,
|
||||
device_map="auto",
|
||||
use_cache=False,
|
||||
)
|
||||
|
||||
print("Adding LoRA...")
|
||||
lora_config = LoraConfig(
|
||||
r=64,
|
||||
lora_alpha=128,
|
||||
target_modules=["q_proj","k_proj","v_proj","o_proj","gate_proj","up_proj"],
|
||||
lora_dropout=0.05,
|
||||
task_type="CAUSAL_LM",
|
||||
)
|
||||
model = get_peft_model(model, lora_config)
|
||||
trainable, total = model.get_nb_trainable_parameters()
|
||||
print(f"Trainable: {trainable:,} / {total:,} ({100*trainable/total:.2f}%)")
|
||||
|
||||
print("Loading dataset...")
|
||||
dataset = load_dataset(
|
||||
"Chamaka8/Serendip-sft-sinhala",
|
||||
data_files={"train": "serendipllm_sft_final_train_v2.json"}
|
||||
)
|
||||
print(f"Examples: {len(dataset['train']):,}")
|
||||
|
||||
def tokenize(examples):
|
||||
texts = []
|
||||
for i in range(len(examples['instruction'])):
|
||||
inp = examples['input'][i] if examples['input'][i] else ""
|
||||
if inp.strip():
|
||||
text = f"### Instruction:\n{examples['instruction'][i]}\n\n### Input:\n{inp}\n\n### Response:\n{examples['output'][i]}"
|
||||
else:
|
||||
text = f"### Instruction:\n{examples['instruction'][i]}\n\n### Response:\n{examples['output'][i]}"
|
||||
texts.append(text)
|
||||
return tokenizer(texts, truncation=True, max_length=512, padding=False)
|
||||
|
||||
print("Tokenizing...")
|
||||
train = dataset["train"].map(
|
||||
tokenize, batched=True, batch_size=5000,
|
||||
num_proc=8, remove_columns=dataset["train"].column_names
|
||||
)
|
||||
|
||||
collator = DataCollatorForLanguageModeling(tokenizer=tokenizer, mlm=False)
|
||||
|
||||
args = TrainingArguments(
|
||||
output_dir=OUTPUT_DIR,
|
||||
num_train_epochs=3,
|
||||
per_device_train_batch_size=8,
|
||||
gradient_accumulation_steps=4,
|
||||
learning_rate=2e-5,
|
||||
warmup_steps=200,
|
||||
weight_decay=0.01,
|
||||
fp16=True,
|
||||
optim="adamw_torch_fused",
|
||||
logging_steps=50,
|
||||
save_steps=2000,
|
||||
save_total_limit=1,
|
||||
eval_strategy="no",
|
||||
dataloader_num_workers=4,
|
||||
gradient_checkpointing=False,
|
||||
report_to="none",
|
||||
)
|
||||
|
||||
trainer = Trainer(
|
||||
model=model,
|
||||
args=args,
|
||||
train_dataset=train,
|
||||
data_collator=collator,
|
||||
)
|
||||
|
||||
print("\n" + "="*70)
|
||||
print("STARTING TRAINING!")
|
||||
print("3 epochs x 309K examples")
|
||||
print("Estimated time: 21 hours")
|
||||
print("="*70 + "\n")
|
||||
|
||||
trainer.train()
|
||||
|
||||
print("\nSaving checkpoint...")
|
||||
trainer.save_model(OUTPUT_DIR + "/checkpoint")
|
||||
tokenizer.save_pretrained(OUTPUT_DIR + "/checkpoint")
|
||||
|
||||
print("Merging LoRA...")
|
||||
model = model.merge_and_unload()
|
||||
|
||||
print("Saving merged model...")
|
||||
model.save_pretrained(OUTPUT_DIR + "/merged")
|
||||
tokenizer.save_pretrained(OUTPUT_DIR + "/merged")
|
||||
|
||||
print("Uploading to HuggingFace...")
|
||||
try:
|
||||
model.push_to_hub(FINAL_MODEL, commit_message="SerendipLLM v2 - Fixed dataset + 3 epochs")
|
||||
tokenizer.push_to_hub(FINAL_MODEL)
|
||||
print(f"Done! https://huggingface.co/{FINAL_MODEL}")
|
||||
except Exception as e:
|
||||
print(f"Upload failed: {e}")
|
||||
print(f"Model saved locally: {OUTPUT_DIR}/merged")
|
||||
|
||||
print("\n" + "="*70)
|
||||
print("TRAINING COMPLETE!")
|
||||
print("="*70)
|
||||
127
training_scripts/train_v2_fast.py
Normal file
127
training_scripts/train_v2_fast.py
Normal file
@@ -0,0 +1,127 @@
|
||||
import torch, os, gc
|
||||
from transformers import AutoModelForCausalLM, AutoTokenizer, TrainingArguments, Trainer, DataCollatorForLanguageModeling
|
||||
from datasets import load_dataset
|
||||
from peft import LoraConfig, get_peft_model
|
||||
|
||||
os.environ["TOKENIZERS_PARALLELISM"] = "false"
|
||||
|
||||
print("="*70)
|
||||
print("SERENDIPLLM V2 - OPTIMIZED (21 HOURS)")
|
||||
print("="*70)
|
||||
|
||||
BASE_MODEL = "Chamaka8/serendib-llm-cpt-llama3-8b"
|
||||
OUTPUT_DIR = "./SerendipLLM-V2"
|
||||
FINAL_MODEL = "Chamaka8/Serendip-LLM-CPT-SFT-v2"
|
||||
|
||||
gc.collect()
|
||||
torch.cuda.empty_cache()
|
||||
|
||||
print("Loading tokenizer...")
|
||||
tokenizer = AutoTokenizer.from_pretrained(BASE_MODEL, use_fast=True)
|
||||
tokenizer.pad_token = tokenizer.eos_token
|
||||
|
||||
print("Loading model...")
|
||||
model = AutoModelForCausalLM.from_pretrained(
|
||||
BASE_MODEL,
|
||||
torch_dtype=torch.float16,
|
||||
device_map="auto",
|
||||
use_cache=False,
|
||||
)
|
||||
|
||||
print("Adding LoRA...")
|
||||
lora_config = LoraConfig(
|
||||
r=64,
|
||||
lora_alpha=128,
|
||||
target_modules=["q_proj","k_proj","v_proj","o_proj","gate_proj","up_proj"],
|
||||
lora_dropout=0.05,
|
||||
task_type="CAUSAL_LM",
|
||||
)
|
||||
model = get_peft_model(model, lora_config)
|
||||
trainable, total = model.get_nb_trainable_parameters()
|
||||
print(f"Trainable: {trainable:,} / {total:,} ({100*trainable/total:.2f}%)")
|
||||
|
||||
print("Loading dataset...")
|
||||
dataset = load_dataset(
|
||||
"Chamaka8/Serendip-sft-sinhala",
|
||||
data_files={"train": "serendipllm_sft_final_train_v2.json"}
|
||||
)
|
||||
print(f"Examples: {len(dataset['train']):,}")
|
||||
|
||||
def tokenize(examples):
|
||||
texts = []
|
||||
for i in range(len(examples['instruction'])):
|
||||
inp = examples['input'][i] if examples['input'][i] else ""
|
||||
if inp.strip():
|
||||
text = f"### Instruction:\n{examples['instruction'][i]}\n\n### Input:\n{inp}\n\n### Response:\n{examples['output'][i]}"
|
||||
else:
|
||||
text = f"### Instruction:\n{examples['instruction'][i]}\n\n### Response:\n{examples['output'][i]}"
|
||||
texts.append(text)
|
||||
return tokenizer(texts, truncation=True, max_length=384, padding=False)
|
||||
|
||||
print("Tokenizing...")
|
||||
train = dataset["train"].map(
|
||||
tokenize, batched=True, batch_size=5000,
|
||||
num_proc=8, remove_columns=dataset["train"].column_names
|
||||
)
|
||||
|
||||
collator = DataCollatorForLanguageModeling(tokenizer=tokenizer, mlm=False)
|
||||
|
||||
args = TrainingArguments(
|
||||
output_dir=OUTPUT_DIR,
|
||||
num_train_epochs=3,
|
||||
per_device_train_batch_size=8,
|
||||
gradient_accumulation_steps=4,
|
||||
learning_rate=2e-5,
|
||||
warmup_steps=200,
|
||||
weight_decay=0.01,
|
||||
fp16=True,
|
||||
optim="adamw_torch_fused",
|
||||
logging_steps=50,
|
||||
save_steps=2000,
|
||||
save_total_limit=1,
|
||||
eval_strategy="no",
|
||||
dataloader_num_workers=4,
|
||||
gradient_checkpointing=False,
|
||||
report_to="none",
|
||||
)
|
||||
|
||||
trainer = Trainer(
|
||||
model=model,
|
||||
args=args,
|
||||
train_dataset=train,
|
||||
data_collator=collator,
|
||||
)
|
||||
|
||||
print("\n" + "="*70)
|
||||
print("STARTING OPTIMIZED TRAINING!")
|
||||
print("max_length: 384 (was 512)")
|
||||
print("Expected speed: ~2.9s/step")
|
||||
print("Expected time: ~21 hours")
|
||||
print("Expected cost: ~$19")
|
||||
print("="*70 + "\n")
|
||||
|
||||
trainer.train()
|
||||
|
||||
print("\nSaving checkpoint...")
|
||||
trainer.save_model(OUTPUT_DIR + "/checkpoint")
|
||||
tokenizer.save_pretrained(OUTPUT_DIR + "/checkpoint")
|
||||
|
||||
print("Merging LoRA...")
|
||||
model = model.merge_and_unload()
|
||||
|
||||
print("Saving merged model...")
|
||||
model.save_pretrained(OUTPUT_DIR + "/merged")
|
||||
tokenizer.save_pretrained(OUTPUT_DIR + "/merged")
|
||||
|
||||
print("Uploading to HuggingFace...")
|
||||
try:
|
||||
model.push_to_hub(FINAL_MODEL, commit_message="SerendipLLM v2 - Fixed dataset + 3 epochs")
|
||||
tokenizer.push_to_hub(FINAL_MODEL)
|
||||
print(f"Done! https://huggingface.co/{FINAL_MODEL}")
|
||||
except Exception as e:
|
||||
print(f"Upload failed: {e}")
|
||||
print(f"Model saved locally: {OUTPUT_DIR}/merged")
|
||||
|
||||
print("\n" + "="*70)
|
||||
print("COMPLETE! SerendipLLM V2 ready!")
|
||||
print("="*70)
|
||||
Reference in New Issue
Block a user