run_name: l20-edu-135m-deepthin
output_dir: runs/l20-edu-135m-deepthin
seed: 1337
tokenizer_name: HuggingFaceTB/SmolLM2-135M
dataset:
  name: HuggingFaceFW/fineweb-edu
  config_name: sample-10BT
  split: train
  streaming: true
  text_column: text
  min_chars: 300
  max_chars: 50000
  min_score: 3.0
  min_int_score: 3
  append_eos: true
  shuffle_buffer: 10000
  max_docs: null
  local_text_path: null
model:
  block_size: 2048
  hidden_size: 576
  intermediate_size: 1536
  num_hidden_layers: 30
  num_attention_heads: 9
  num_key_value_heads: 3
  rope_theta: 10000.0
  rms_norm_eps: 1.0e-06
  attention_dropout: 0.0
  tie_word_embeddings: true
  vocab_multiple: 64
  attn_implementation: sdpa
trainer:
  micro_batch_size: 6
  gradient_accumulation_steps: 43
  max_steps: 18928
  warmup_steps: 1000
  learning_rate: 0.0004
  min_lr_ratio: 0.1
  weight_decay: 0.1
  beta1: 0.9
  beta2: 0.95
  grad_clip: 1.0
  dtype: bfloat16
  compile: true
  gradient_checkpointing: true
  log_interval: 10
  eval_interval: 500
  eval_batches: 64
  save_interval: 1000
  keep_last_checkpoints: 2
  num_workers: 0