初始化项目，由ModelHub XC社区提供模型

Model: KookiesXy/Neo50M Source: Original Platform
2026-07-17 09:27:11 +08:00
commit 5f250fbae2
22 changed files with 277683 additions and 0 deletions
--- a/training_metadata.json
+++ b/training_metadata.json
@@ -0,0 +1,96 @@
+{
+  "best_val_loss": 2.788245379924774,
+  "config": {
+    "_config_path": "configs/sft_chat.yaml",
+    "run": {
+      "data_config": "configs/data_mix.yaml",
+      "model_config": "configs/model_neo50m.yaml",
+      "name": "neo50m_sft_chat",
+      "output_dir": "outputs",
+      "seed": 3337
+    },
+    "training": {
+      "beta1": 0.9,
+      "beta2": 0.95,
+      "curriculum": [
+        {
+          "seq_len": 4096,
+          "tokens": 40000000
+        },
+        {
+          "seq_len": 8192,
+          "tokens": 25000000
+        },
+        {
+          "seq_len": 16384,
+          "tokens": 15000000
+        }
+      ],
+      "data_mode": "sft",
+      "eval_batches": 4,
+      "eval_interval_steps": 250,
+      "grad_accum": 8,
+      "grad_clip": 1.0,
+      "keep_last_checkpoints": 2,
+      "learning_rate": 8e-05,
+      "log_interval_steps": 5,
+      "micro_batch_size": 1,
+      "min_lr_ratio": 0.1,
+      "num_workers": 0,
+      "save_interval_steps": 250,
+      "seq_len": 4096,
+      "stage": "sft_chat",
+      "target_tokens": 80000000,
+      "warmup_steps": 200,
+      "weight_decay": 0.0
+    }
+  },
+  "env": {
+    "cuda": "13.0",
+    "git_revision": null,
+    "gpu_count": 8,
+    "gpu_name": "NVIDIA GeForce RTX 5090",
+    "python": "3.12.13",
+    "torch": "2.12.0+cu130"
+  },
+  "global_step": 57750,
+  "global_tokens": 15628763136,
+  "model_config": {
+    "architecture": "llama",
+    "attention_bias": false,
+    "attention_dropout": 0.0,
+    "attn_implementation": "sdpa",
+    "gradient_checkpointing": false,
+    "hidden_size": 512,
+    "initializer_range": 0.02,
+    "intermediate_size": 1536,
+    "max_position_embeddings": 16384,
+    "mlp_bias": false,
+    "name": "Neo50M",
+    "num_attention_heads": 8,
+    "num_hidden_layers": 12,
+    "num_key_value_heads": 2,
+    "rms_norm_eps": 1e-06,
+    "rope_theta": 500000.0,
+    "tie_word_embeddings": true,
+    "tokenizer_id": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
+    "torch_dtype": "bfloat16",
+    "use_cache": false
+  },
+  "precision": {
+    "fp8_backend": null,
+    "reason": "FP8 unavailable; torchao is not installed",
+    "requested": "auto",
+    "selected": "bf16"
+  },
+  "stage_id": "sft_chat_0_seq4096",
+  "stage_tokens": 25690112,
+  "tokenizer": {
+    "bos_token_id": 1,
+    "chat_template": "{% for message in messages %}{% if loop.first and message['role'] != 'system' %}{{ bos_token + 'System: You are Neo50M, a concise and helpful assistant.\\n' }}{% endif %}{% if message['role'] == 'system' %}{{ bos_token + 'System: ' + message['content'].strip() + '\\n' }}{% elif message['role'] == 'user' %}{{ 'User: ' + message['content'].strip() + '\\n' }}{% elif message['role'] == 'assistant' %}{{ 'Assistant: ' + message['content'].strip() + eos_token }}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ 'Assistant: ' }}{% endif %}",
+    "eos_token_id": 2,
+    "pad_token_id": 2,
+    "tokenizer_id": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
+    "vocab_size": 32000
+  }
+}