32 lines
640 B
YAML
32 lines
640 B
YAML
|
run:
|
||
|
|
name: neo50m_long_context
|
||
|
|
output_dir: outputs
|
||
|
|
model_config: configs/model_neo50m.yaml
|
||
|
|
data_config: configs/data_mix.yaml
|
||
|
|
seed: 2337
|
||
|
|
|
||
|
|
training:
|
||
|
|
stage: long_context
|
||
|
|
target_tokens: 600000000
|
||
|
|
micro_batch_size: 1
|
||
|
|
grad_accum: 16
|
||
|
|
learning_rate: 0.0002
|
||
|
|
min_lr_ratio: 0.2
|
||
|
|
warmup_steps: 500
|
||
|
|
weight_decay: 0.1
|
||
|
|
beta1: 0.9
|
||
|
|
beta2: 0.95
|
||
|
|
grad_clip: 1.0
|
||
|
|
eval_interval_steps: 500
|
||
|
|
eval_batches: 8
|
||
|
|
save_interval_steps: 500
|
||
|
|
keep_last_checkpoints: 2
|
||
|
|
log_interval_steps: 5
|
||
|
|
num_workers: 0
|
||
|
|
data_mode: streaming
|
||
|
|
curriculum:
|
||
|
|
- seq_len: 8192
|
||
|
|
tokens: 300000000
|
||
|
|
- seq_len: 16384
|
||
|
|
tokens: 300000000
|