run_name: l20-edu-135m-stage2-math-code-textbook-replay-8k
output_dir: runs/l20-edu-135m-stage2-math-code-textbook-replay-8k
seed: 20260610
tokenizer_name: AliceYin/l20-edu-135m
init_model_name_or_path: runs/l20-edu-135m-hq-longctx-8k/final
dataset:
  name: HuggingFaceFW/fineweb-edu
  config_name: sample-10BT
  split: train
  streaming: true
  text_column: text
  tokenized_path: data/l20_stage2_math_code_textbook_replay_8k
  min_chars: 200
  max_chars: 50000
  min_score: null
  min_int_score: null
  append_eos: true
  shuffle_buffer: 10000
  max_docs: null
  local_text_path: null
model:
  block_size: 8192
  hidden_size: 576
  intermediate_size: 1536
  num_hidden_layers: 30
  num_attention_heads: 9
  num_key_value_heads: 3
  rope_theta: 10000.0
  rms_norm_eps: 1.0e-06
  attention_dropout: 0.0
  tie_word_embeddings: true
  vocab_multiple: 64
  attn_implementation: sdpa
  rope_scaling:
    rope_type: yarn
    factor: 4.0
    original_max_position_embeddings: 2048
trainer:
  micro_batch_size: 3
  gradient_accumulation_steps: 22
  max_steps: 1850
  warmup_steps: 100
  learning_rate: 1.0e-05
  min_lr_ratio: 0.15
  weight_decay: 0.1
  beta1: 0.9
  beta2: 0.95
  grad_clip: 1.0
  dtype: bfloat16
  compile: false
  compile_mode: null
  compile_fullgraph: null
  liger_kernel: true
  gradient_checkpointing: false
  log_interval: 1
  eval_interval: 100
  eval_batches: 8
  save_interval: 100
  keep_last_checkpoints: 4
  num_workers: 0
  mfu_peak_tflops: 119.5