初始化项目，由ModelHub XC社区提供模型

Model: AliceYin/l20-edu-135m
Source: Original Platform

pretrain_config.yaml

@@ -0,0 +1,51 @@
+run_name: l20-edu-135m-deepthin
+output_dir: runs/l20-edu-135m-deepthin
+seed: 1337
+tokenizer_name: HuggingFaceTB/SmolLM2-135M
+dataset:
+  name: HuggingFaceFW/fineweb-edu
+  config_name: sample-10BT
+  split: train
+  streaming: true
+  text_column: text
+  min_chars: 300
+  max_chars: 50000
+  min_score: 3.0
+  min_int_score: 3
+  append_eos: true
+  shuffle_buffer: 10000
+  max_docs: null
+  local_text_path: null
+model:
+  block_size: 2048
+  hidden_size: 576
+  intermediate_size: 1536
+  num_hidden_layers: 30
+  num_attention_heads: 9
+  num_key_value_heads: 3
+  rope_theta: 10000.0
+  rms_norm_eps: 1.0e-06
+  attention_dropout: 0.0
+  tie_word_embeddings: true
+  vocab_multiple: 64
+  attn_implementation: sdpa
+trainer:
+  micro_batch_size: 6
+  gradient_accumulation_steps: 43
+  max_steps: 18928
+  warmup_steps: 1000
+  learning_rate: 0.0004
+  min_lr_ratio: 0.1
+  weight_decay: 0.1
+  beta1: 0.9
+  beta2: 0.95
+  grad_clip: 1.0
+  dtype: bfloat16
+  compile: true
+  gradient_checkpointing: true
+  log_interval: 10
+  eval_interval: 500
+  eval_batches: 64
+  save_interval: 1000
+  keep_last_checkpoints: 2
+  num_workers: 0