初始化项目，由ModelHub XC社区提供模型

Model: KookiesXy/Neo50M
Source: Original Platform

training_configs/data_mix.yaml

@@ -0,0 +1,85 @@
+pretrain:
+  shuffle_buffer: 10000
+  seed: 1337
+  allow_synthetic_fallback: false
+  max_doc_tokens: 16384
+  max_chars_per_doc: 120000
+  sources:
+    - name: fineweb_edu
+      path: HuggingFaceFW/fineweb-edu
+      config: sample-10BT
+      split: train
+      text_field: text
+      weight: 0.0
+      streaming: true
+      fail_on_error: false
+    - name: cosmopedia
+      path: HuggingFaceTB/cosmopedia
+      config: web_samples_v2
+      split: train
+      text_field: text
+      weight: 0.0
+      streaming: true
+      fail_on_error: false
+    - name: wikipedia
+      path: wikimedia/wikipedia
+      config: 20231101.en
+      split: train
+      text_field: text
+      weight: 0.0
+      streaming: true
+      fail_on_error: false
+    - name: tinystories
+      path: roneneldan/TinyStories
+      split: train
+      text_field: text
+      weight: 0.75
+      streaming: true
+      fail_on_error: false
+    - name: permissive_code
+      path: nuprl/stack-dedup-python-testgen-starcoder-filter-v2
+      split: train
+      text_field: content
+      weight: 0.25
+      streaming: true
+      fail_on_error: false
+
+validation:
+  sources:
+    - name: wikitext103
+      path: Salesforce/wikitext
+      config: wikitext-103-raw-v1
+      split: validation
+      text_field: text
+      weight: 1.0
+      streaming: true
+      fail_on_error: false
+
+sft:
+  shuffle_buffer: 5000
+  seed: 2025
+  assistant_only_loss: true
+  allow_synthetic_fallback: true
+  sources:
+    - name: openhermes
+      path: teknium/OpenHermes-2.5
+      split: train
+      weight: 0.45
+      streaming: false
+      fail_on_error: false
+    - name: ultrachat
+      path: HuggingFaceH4/ultrachat_200k
+      split: train_sft
+      weight: 0.35
+      streaming: false
+      fail_on_error: false
+    - name: alpaca
+      path: tatsu-lab/alpaca
+      split: train
+      weight: 0.15
+      streaming: false
+      fail_on_error: false
+    - name: safety_small
+      synthetic: safety_refusals
+      weight: 0.05
+      fail_on_error: false

training_configs/export.yaml

@@ -0,0 +1,12 @@
+export:
+  repo_id: KookiesXy/Neo50M
+  checkpoint: outputs/checkpoints/best
+  hf_dir: outputs/hf
+  onnx_dir: outputs/onnx
+  gguf_dir: outputs/gguf
+  eval_dir: outputs/evals
+  max_shard_size: 2GB
+  gguf_outtypes:
+    - f16
+    - q8_0
+    - q4_k_m

training_configs/long_context.yaml

@@ -0,0 +1,31 @@
+run:
+  name: neo50m_long_context
+  output_dir: outputs
+  model_config: configs/model_neo50m.yaml
+  data_config: configs/data_mix.yaml
+  seed: 2337
+
+training:
+  stage: long_context
+  target_tokens: 600000000
+  micro_batch_size: 1
+  grad_accum: 16
+  learning_rate: 0.0002
+  min_lr_ratio: 0.2
+  warmup_steps: 500
+  weight_decay: 0.1
+  beta1: 0.9
+  beta2: 0.95
+  grad_clip: 1.0
+  eval_interval_steps: 500
+  eval_batches: 8
+  save_interval_steps: 500
+  keep_last_checkpoints: 2
+  log_interval_steps: 5
+  num_workers: 0
+  data_mode: streaming
+  curriculum:
+    - seq_len: 8192
+      tokens: 300000000
+    - seq_len: 16384
+      tokens: 300000000

training_configs/model_neo50m.yaml

@@ -0,0 +1,21 @@
+model:
+  name: Neo50M
+  architecture: llama
+  tokenizer_id: TinyLlama/TinyLlama-1.1B-Chat-v1.0
+  hidden_size: 512
+  intermediate_size: 1536
+  num_hidden_layers: 12
+  num_attention_heads: 8
+  num_key_value_heads: 2
+  max_position_embeddings: 16384
+  rope_theta: 500000.0
+  rms_norm_eps: 0.000001
+  initializer_range: 0.02
+  tie_word_embeddings: true
+  use_cache: false
+  attention_dropout: 0.0
+  attention_bias: false
+  mlp_bias: false
+  attn_implementation: sdpa
+  gradient_checkpointing: false
+  torch_dtype: bfloat16

training_configs/pretrain_15b.yaml

@@ -0,0 +1,27 @@
+run:
+  name: neo50m_pretrain_15b
+  output_dir: outputs
+  model_config: configs/model_neo50m.yaml
+  data_config: configs/data_mix.yaml
+  seed: 1337
+
+training:
+  stage: pretrain_4k
+  seq_len: 2048
+  target_tokens: 15000000000
+  micro_batch_size: 4
+  grad_accum: 4
+  learning_rate: 0.0008
+  min_lr_ratio: 0.1
+  warmup_steps: 2000
+  weight_decay: 0.1
+  beta1: 0.9
+  beta2: 0.95
+  grad_clip: 1.0
+  eval_interval_steps: 1000
+  eval_batches: 16
+  save_interval_steps: 1000
+  keep_last_checkpoints: 2
+  log_interval_steps: 10
+  num_workers: 0
+  data_mode: streaming

training_configs/pretrain_fast.yaml

@@ -0,0 +1,27 @@
+run:
+  name: neo50m_smoke
+  output_dir: outputs/smoke
+  model_config: configs/model_neo50m.yaml
+  data_config: configs/data_mix.yaml
+  seed: 1337
+
+training:
+  stage: smoke
+  seq_len: 256
+  target_tokens: 1000000
+  micro_batch_size: 2
+  grad_accum: 1
+  learning_rate: 0.0006
+  min_lr_ratio: 0.1
+  warmup_steps: 20
+  weight_decay: 0.1
+  beta1: 0.9
+  beta2: 0.95
+  grad_clip: 1.0
+  eval_interval_steps: 0
+  eval_batches: 2
+  save_interval_steps: 100
+  keep_last_checkpoints: 2
+  log_interval_steps: 10
+  num_workers: 0
+  data_mode: random

training_configs/sft_chat.yaml

@@ -0,0 +1,34 @@
+run:
+  name: neo50m_sft_chat
+  output_dir: outputs
+  model_config: configs/model_neo50m.yaml
+  data_config: configs/data_mix.yaml
+  seed: 3337
+
+training:
+  stage: sft_chat
+  seq_len: 4096
+  target_tokens: 80000000
+  micro_batch_size: 1
+  grad_accum: 8
+  learning_rate: 0.00008
+  min_lr_ratio: 0.1
+  warmup_steps: 200
+  weight_decay: 0.0
+  beta1: 0.9
+  beta2: 0.95
+  grad_clip: 1.0
+  eval_interval_steps: 250
+  eval_batches: 4
+  save_interval_steps: 250
+  keep_last_checkpoints: 2
+  log_interval_steps: 5
+  num_workers: 0
+  data_mode: sft
+  curriculum:
+    - seq_len: 4096
+      tokens: 40000000
+    - seq_len: 8192
+      tokens: 25000000
+    - seq_len: 16384
+      tokens: 15000000