28 lines
536 B
YAML
28 lines
536 B
YAML
|
run:
|
||
|
|
name: neo50m_smoke
|
||
|
|
output_dir: outputs/smoke
|
||
|
|
model_config: configs/model_neo50m.yaml
|
||
|
|
data_config: configs/data_mix.yaml
|
||
|
|
seed: 1337
|
||
|
|
|
||
|
|
training:
|
||
|
|
stage: smoke
|
||
|
|
seq_len: 256
|
||
|
|
target_tokens: 1000000
|
||
|
|
micro_batch_size: 2
|
||
|
|
grad_accum: 1
|
||
|
|
learning_rate: 0.0006
|
||
|
|
min_lr_ratio: 0.1
|
||
|
|
warmup_steps: 20
|
||
|
|
weight_decay: 0.1
|
||
|
|
beta1: 0.9
|
||
|
|
beta2: 0.95
|
||
|
|
grad_clip: 1.0
|
||
|
|
eval_interval_steps: 0
|
||
|
|
eval_batches: 2
|
||
|
|
save_interval_steps: 100
|
||
|
|
keep_last_checkpoints: 2
|
||
|
|
log_interval_steps: 10
|
||
|
|
num_workers: 0
|
||
|
|
data_mode: random
|