run:
  name: neo50m_sft_chat
  output_dir: outputs
  model_config: configs/model_neo50m.yaml
  data_config: configs/data_mix.yaml
  seed: 3337

training:
  stage: sft_chat
  seq_len: 4096
  target_tokens: 80000000
  micro_batch_size: 1
  grad_accum: 8
  learning_rate: 0.00008
  min_lr_ratio: 0.1
  warmup_steps: 200
  weight_decay: 0.0
  beta1: 0.9
  beta2: 0.95
  grad_clip: 1.0
  eval_interval_steps: 250
  eval_batches: 4
  save_interval_steps: 250
  keep_last_checkpoints: 2
  log_interval_steps: 5
  num_workers: 0
  data_mode: sft
  curriculum:
    - seq_len: 4096
      tokens: 40000000
    - seq_len: 8192
      tokens: 25000000
    - seq_len: 16384
      tokens: 15000000