run:
  name: neo50m_long_context
  output_dir: outputs
  model_config: configs/model_neo50m.yaml
  data_config: configs/data_mix.yaml
  seed: 2337

training:
  stage: long_context
  target_tokens: 600000000
  micro_batch_size: 1
  grad_accum: 16
  learning_rate: 0.0002
  min_lr_ratio: 0.2
  warmup_steps: 500
  weight_decay: 0.1
  beta1: 0.9
  beta2: 0.95
  grad_clip: 1.0
  eval_interval_steps: 500
  eval_batches: 8
  save_interval_steps: 500
  keep_last_checkpoints: 2
  log_interval_steps: 5
  num_workers: 0
  data_mode: streaming
  curriculum:
    - seq_len: 8192
      tokens: 300000000
    - seq_len: 16384
      tokens: 300000000