run:
  name: neo50m_pretrain_15b
  output_dir: outputs
  model_config: configs/model_neo50m.yaml
  data_config: configs/data_mix.yaml
  seed: 1337

training:
  stage: pretrain_4k
  seq_len: 2048
  target_tokens: 15000000000
  micro_batch_size: 4
  grad_accum: 4
  learning_rate: 0.0008
  min_lr_ratio: 0.1
  warmup_steps: 2000
  weight_decay: 0.1
  beta1: 0.9
  beta2: 0.95
  grad_clip: 1.0
  eval_interval_steps: 1000
  eval_batches: 16
  save_interval_steps: 1000
  keep_last_checkpoints: 2
  log_interval_steps: 10
  num_workers: 0
  data_mode: streaming