62 lines
1.4 KiB
YAML
62 lines
1.4 KiB
YAML
|
run_name: l20-edu-135m-stage2-math-code-textbook-replay-8k
|
||
|
|
output_dir: runs/l20-edu-135m-stage2-math-code-textbook-replay-8k
|
||
|
|
seed: 20260610
|
||
|
|
tokenizer_name: AliceYin/l20-edu-135m
|
||
|
|
init_model_name_or_path: runs/l20-edu-135m-hq-longctx-8k/final
|
||
|
|
dataset:
|
||
|
|
name: HuggingFaceFW/fineweb-edu
|
||
|
|
config_name: sample-10BT
|
||
|
|
split: train
|
||
|
|
streaming: true
|
||
|
|
text_column: text
|
||
|
|
tokenized_path: data/l20_stage2_math_code_textbook_replay_8k
|
||
|
|
min_chars: 200
|
||
|
|
max_chars: 50000
|
||
|
|
min_score: null
|
||
|
|
min_int_score: null
|
||
|
|
append_eos: true
|
||
|
|
shuffle_buffer: 10000
|
||
|
|
max_docs: null
|
||
|
|
local_text_path: null
|
||
|
|
model:
|
||
|
|
block_size: 8192
|
||
|
|
hidden_size: 576
|
||
|
|
intermediate_size: 1536
|
||
|
|
num_hidden_layers: 30
|
||
|
|
num_attention_heads: 9
|
||
|
|
num_key_value_heads: 3
|
||
|
|
rope_theta: 10000.0
|
||
|
|
rms_norm_eps: 1.0e-06
|
||
|
|
attention_dropout: 0.0
|
||
|
|
tie_word_embeddings: true
|
||
|
|
vocab_multiple: 64
|
||
|
|
attn_implementation: sdpa
|
||
|
|
rope_scaling:
|
||
|
|
rope_type: yarn
|
||
|
|
factor: 4.0
|
||
|
|
original_max_position_embeddings: 2048
|
||
|
|
trainer:
|
||
|
|
micro_batch_size: 3
|
||
|
|
gradient_accumulation_steps: 22
|
||
|
|
max_steps: 1850
|
||
|
|
warmup_steps: 100
|
||
|
|
learning_rate: 1.0e-05
|
||
|
|
min_lr_ratio: 0.15
|
||
|
|
weight_decay: 0.1
|
||
|
|
beta1: 0.9
|
||
|
|
beta2: 0.95
|
||
|
|
grad_clip: 1.0
|
||
|
|
dtype: bfloat16
|
||
|
|
compile: false
|
||
|
|
compile_mode: null
|
||
|
|
compile_fullgraph: null
|
||
|
|
liger_kernel: true
|
||
|
|
gradient_checkpointing: false
|
||
|
|
log_interval: 1
|
||
|
|
eval_interval: 100
|
||
|
|
eval_batches: 8
|
||
|
|
save_interval: 100
|
||
|
|
keep_last_checkpoints: 4
|
||
|
|
num_workers: 0
|
||
|
|
mfu_peak_tflops: 119.5
|