28 lines
558 B
YAML
28 lines
558 B
YAML
run:
|
|
name: neo50m_pretrain_15b
|
|
output_dir: outputs
|
|
model_config: configs/model_neo50m.yaml
|
|
data_config: configs/data_mix.yaml
|
|
seed: 1337
|
|
|
|
training:
|
|
stage: pretrain_4k
|
|
seq_len: 2048
|
|
target_tokens: 15000000000
|
|
micro_batch_size: 4
|
|
grad_accum: 4
|
|
learning_rate: 0.0008
|
|
min_lr_ratio: 0.1
|
|
warmup_steps: 2000
|
|
weight_decay: 0.1
|
|
beta1: 0.9
|
|
beta2: 0.95
|
|
grad_clip: 1.0
|
|
eval_interval_steps: 1000
|
|
eval_batches: 16
|
|
save_interval_steps: 1000
|
|
keep_last_checkpoints: 2
|
|
log_interval_steps: 10
|
|
num_workers: 0
|
|
data_mode: streaming
|