35 lines
682 B
YAML
35 lines
682 B
YAML
|
run:
|
||
|
|
name: neo50m_sft_chat
|
||
|
|
output_dir: outputs
|
||
|
|
model_config: configs/model_neo50m.yaml
|
||
|
|
data_config: configs/data_mix.yaml
|
||
|
|
seed: 3337
|
||
|
|
|
||
|
|
training:
|
||
|
|
stage: sft_chat
|
||
|
|
seq_len: 4096
|
||
|
|
target_tokens: 80000000
|
||
|
|
micro_batch_size: 1
|
||
|
|
grad_accum: 8
|
||
|
|
learning_rate: 0.00008
|
||
|
|
min_lr_ratio: 0.1
|
||
|
|
warmup_steps: 200
|
||
|
|
weight_decay: 0.0
|
||
|
|
beta1: 0.9
|
||
|
|
beta2: 0.95
|
||
|
|
grad_clip: 1.0
|
||
|
|
eval_interval_steps: 250
|
||
|
|
eval_batches: 4
|
||
|
|
save_interval_steps: 250
|
||
|
|
keep_last_checkpoints: 2
|
||
|
|
log_interval_steps: 5
|
||
|
|
num_workers: 0
|
||
|
|
data_mode: sft
|
||
|
|
curriculum:
|
||
|
|
- seq_len: 4096
|
||
|
|
tokens: 40000000
|
||
|
|
- seq_len: 8192
|
||
|
|
tokens: 25000000
|
||
|
|
- seq_len: 16384
|
||
|
|
tokens: 15000000
|