52 lines
1.1 KiB
YAML
52 lines
1.1 KiB
YAML
|
run_name: l20-edu-135m-deepthin
|
||
|
|
output_dir: runs/l20-edu-135m-deepthin
|
||
|
|
seed: 1337
|
||
|
|
tokenizer_name: HuggingFaceTB/SmolLM2-135M
|
||
|
|
dataset:
|
||
|
|
name: HuggingFaceFW/fineweb-edu
|
||
|
|
config_name: sample-10BT
|
||
|
|
split: train
|
||
|
|
streaming: true
|
||
|
|
text_column: text
|
||
|
|
min_chars: 300
|
||
|
|
max_chars: 50000
|
||
|
|
min_score: 3.0
|
||
|
|
min_int_score: 3
|
||
|
|
append_eos: true
|
||
|
|
shuffle_buffer: 10000
|
||
|
|
max_docs: null
|
||
|
|
local_text_path: null
|
||
|
|
model:
|
||
|
|
block_size: 2048
|
||
|
|
hidden_size: 576
|
||
|
|
intermediate_size: 1536
|
||
|
|
num_hidden_layers: 30
|
||
|
|
num_attention_heads: 9
|
||
|
|
num_key_value_heads: 3
|
||
|
|
rope_theta: 10000.0
|
||
|
|
rms_norm_eps: 1.0e-06
|
||
|
|
attention_dropout: 0.0
|
||
|
|
tie_word_embeddings: true
|
||
|
|
vocab_multiple: 64
|
||
|
|
attn_implementation: sdpa
|
||
|
|
trainer:
|
||
|
|
micro_batch_size: 6
|
||
|
|
gradient_accumulation_steps: 43
|
||
|
|
max_steps: 18928
|
||
|
|
warmup_steps: 1000
|
||
|
|
learning_rate: 0.0004
|
||
|
|
min_lr_ratio: 0.1
|
||
|
|
weight_decay: 0.1
|
||
|
|
beta1: 0.9
|
||
|
|
beta2: 0.95
|
||
|
|
grad_clip: 1.0
|
||
|
|
dtype: bfloat16
|
||
|
|
compile: true
|
||
|
|
gradient_checkpointing: true
|
||
|
|
log_interval: 10
|
||
|
|
eval_interval: 500
|
||
|
|
eval_batches: 64
|
||
|
|
save_interval: 1000
|
||
|
|
keep_last_checkpoints: 2
|
||
|
|
num_workers: 0
|