初始化项目,由ModelHub XC社区提供模型
Model: pathcosmos/frankenstallm Source: Original Platform
This commit is contained in:
47
source/configs/korean_3b_sft.yaml
Normal file
47
source/configs/korean_3b_sft.yaml
Normal file
@@ -0,0 +1,47 @@
|
||||
# Korean 3B SFT Configuration
|
||||
#
|
||||
# Base model: checkpoints/korean_3b_fp8_run1/checkpoint-XXXXXX (3B params pretrained)
|
||||
# SFT 목표: instruction following + 반복 퇴화 완화 + 생성 품질 향상
|
||||
# 아키텍처: LLaMA-3 3B 참고 (d=3072, 28L, 24H, GQA 8:1)
|
||||
#
|
||||
# 실행: bash scripts/launch_3b_sft.sh
|
||||
#
|
||||
# [설계 근거 — 2026-03-02]
|
||||
# - batch: 2 × 8GPU × 4 grad_accum = 64 eff_batch
|
||||
# - max_steps 33000 ≈ 3 epochs × 700K samples / 64 eff_batch
|
||||
# - lr=1e-5: pretrain 1.5e-4의 1/15 (catastrophic forgetting 방지)
|
||||
# - NEFTune alpha=5.0: 생성 다양성 향상, 반복 퇴화 완화
|
||||
# - use_fp8=true: B200 MXFP8 네이티브 가속 유지
|
||||
|
||||
model:
|
||||
vocab_size: 64000
|
||||
d_model: 3072
|
||||
n_layers: 28
|
||||
n_heads: 24
|
||||
n_kv_heads: 8
|
||||
d_ffn: 8192
|
||||
max_seq_len: 4096
|
||||
rope_theta: 500000.0
|
||||
dropout: 0.0
|
||||
bias: false
|
||||
use_flash_attn: true
|
||||
use_fp8: true
|
||||
|
||||
train:
|
||||
max_steps: 33000 # 3 epochs × 700K / 64 eff_batch
|
||||
batch_size: 2 # per GPU (3B VRAM 절약)
|
||||
grad_accum_steps: 4 # eff_batch: 2 × 8GPU × 4 = 64
|
||||
lr: 1.0e-5 # catastrophic forgetting 방지
|
||||
weight_decay: 0.01
|
||||
warmup_steps: 500
|
||||
max_grad_norm: 1.0
|
||||
log_interval: 10
|
||||
save_interval: 2000
|
||||
eval_interval: 500
|
||||
use_amp: false
|
||||
compile_model: false
|
||||
neftune_alpha: 5.0 # NEFTune noise injection
|
||||
|
||||
tokenizer:
|
||||
vocab_size: 64000
|
||||
type: sentencepiece_unigram
|
||||
Reference in New Issue
Block a user