50 lines
1.2 KiB
YAML
50 lines
1.2 KiB
YAML
|
# Korean 3B ORPO Configuration (Phase 3)
|
||
|
|
#
|
||
|
|
# Base model: SFT v1 best checkpoint (HF format)
|
||
|
|
# 목표: Greedy 반복률 73%→30% 이하, EOS 종료율 0%→80%+
|
||
|
|
#
|
||
|
|
# 실행:
|
||
|
|
# bash scripts/launch_3b_orpo.sh # 본 학습
|
||
|
|
# bash scripts/launch_3b_orpo.sh --max_steps 200 # 퀵 테스트
|
||
|
|
#
|
||
|
|
# [설계 근거]
|
||
|
|
# - beta=0.25: 반복률 73%는 극단적 → 강한 OR loss 필요 (기존 0.1에서 상향)
|
||
|
|
# - lr=1.2e-5: HP sweep 6-config 결과 최적 (eval_loss 1.862, margin +0.009)
|
||
|
|
# - epochs=2: 683K 규모에 3 epoch은 과적합 위험
|
||
|
|
# - max_length=1536: P95=880 tokens, 99%+ 커버 + VRAM 25% 절약
|
||
|
|
|
||
|
|
# Model
|
||
|
|
model_path: eval/outputs/hf_3b_sft_best
|
||
|
|
output_dir: checkpoints/korean_3b_orpo
|
||
|
|
|
||
|
|
# Training
|
||
|
|
epochs: 2
|
||
|
|
lr: 1.2e-5
|
||
|
|
beta: 0.25
|
||
|
|
batch_size: 4
|
||
|
|
gradient_accumulation_steps: 4
|
||
|
|
max_length: 1536
|
||
|
|
bf16: true
|
||
|
|
weight_decay: 0.01
|
||
|
|
seed: 42
|
||
|
|
|
||
|
|
# Scheduler
|
||
|
|
lr_scheduler_type: cosine
|
||
|
|
warmup_ratio: 0.05
|
||
|
|
|
||
|
|
# Evaluation & Early Stopping
|
||
|
|
eval_split_ratio: 0.05
|
||
|
|
eval_steps: 500
|
||
|
|
early_stopping_patience: 3
|
||
|
|
|
||
|
|
# Logging & checkpointing
|
||
|
|
logging_steps: 10
|
||
|
|
save_steps: 500
|
||
|
|
save_total_limit: 5
|
||
|
|
gradient_checkpointing: true
|
||
|
|
report_to: none
|
||
|
|
|
||
|
|
# Data
|
||
|
|
custom_data_path: data/preference/combined_preference.jsonl
|
||
|
|
dataset_num_proc: 64
|