初始化项目,由ModelHub XC社区提供模型
Model: NyayaLabs98/nyaya-3b-v3 Source: Original Platform
This commit is contained in:
39
training/train_v3.yaml
Normal file
39
training/train_v3.yaml
Normal file
@@ -0,0 +1,39 @@
|
||||
# Nyaya-3B-v3 — RAFT: teacher answers regenerated UNDER the inference RAG
|
||||
# prompt (data/splits_rag_v3, scripts/20+21), including deliberate
|
||||
# retrieval-miss demonstrations. 2 epochs (v2's val loss was still falling
|
||||
# at 1) + NEFTune noise for SFT robustness.
|
||||
#
|
||||
# Method: full-precision bf16 LoRA — NO quantization (project decision 2026-07-13).
|
||||
|
||||
run_name: legal-3b-v3
|
||||
model_id: Qwen/Qwen2.5-3B-Instruct
|
||||
output_dir: outputs/legal-3b-v3
|
||||
|
||||
data:
|
||||
train_file: data/splits_rag_v3/train.jsonl
|
||||
val_file: data/splits_rag_v3/val.jsonl
|
||||
|
||||
method: lora
|
||||
|
||||
lora:
|
||||
r: 32
|
||||
alpha: 64
|
||||
dropout: 0.05
|
||||
target_modules: [q_proj, k_proj, v_proj, o_proj, gate_proj, up_proj, down_proj]
|
||||
|
||||
training:
|
||||
max_seq_length: 6144
|
||||
num_train_epochs: 2
|
||||
learning_rate: 1.0e-4
|
||||
warmup_ratio: 0.03
|
||||
lr_scheduler_type: cosine
|
||||
gradient_checkpointing: true
|
||||
bf16: true
|
||||
optim: adamw_torch_fused
|
||||
neftune_noise_alpha: 5
|
||||
per_device_train_batch_size: 4
|
||||
gradient_accumulation_steps: 8
|
||||
logging_steps: 10
|
||||
# ~6.3K examples x 2 epochs / effective batch 32 = ~394 steps
|
||||
eval_steps: 50
|
||||
save_steps: 100
|
||||
Reference in New Issue
Block a user