初始化项目，由ModelHub XC社区提供模型

Model: FlameF0X/TinyMoE-200m-2x16
Source: Original Platform

.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,56 @@
+---
+license: mit
+datasets:
+- HuggingFaceTB/smollm-corpus
+language:
+- en
+pipeline_tag: text-generation
+library_name: transformers
+tags:
+- moe
+- causal-lm
+- mistral
+- tiny-moe
+---
+
+# TinyMoE-200M-2x16
+
+**TinyMoE-200M-2x16** is a compact, highly efficient Sparse Mixture of Experts (MoE) language model built upon the Mixtral/Mistral architecture. Designed for research, edge applications, and resource-constrained environments, this model leverages an expert-routing mechanism to balance a larger total parameter capacity with ultra-low computational overhead during inference.
+
+## Model Details
+
+* **Architecture:** Sparse Mixture of Experts (MoE)
+* **Total Parameters:** ~200m total parameters
+* **Active Parameters per Token:** ~50M active parameters
+* **Expert Configuration:** 16 total local experts, 2 active experts routed per token (`num_experts_per_tok": 2`)
+* **Context Length:** 1024 tokens
+* **Base Architecture:** Mixtral / Mistral For Causal LM
+* **License:** MIT
+
+## Training Data
+
+This model was trained on`FineWeb-Edu-Dedup` 60% and `Cosmopedia-v2` 40%.
+
+![image](https://cdn-uploads.huggingface.co/production/uploads/6615494716917dfdc645c44e/35iLfOIZI-tKMP2TUJ40E.png)
+![image](https://cdn-uploads.huggingface.co/production/uploads/6615494716917dfdc645c44e/IyrW-dzolURCavH-AD5a2.png)
+![image](https://cdn-uploads.huggingface.co/production/uploads/6615494716917dfdc645c44e/eNsiJy4VXV3zFh2v-qAoF.png)
+![image](https://cdn-uploads.huggingface.co/production/uploads/6615494716917dfdc645c44e/37VEpxpu99tJiTAqPJtcT.png)
+![image](https://cdn-uploads.huggingface.co/production/uploads/6615494716917dfdc645c44e/5Z3Xh9xtFnACgO4-qtamg.png)
+
+## Quick Start
+
+You can load and experiment with this model using the Hugging Face `transformers` library:
+
+```python
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+model_id = "FlameF0X/TinyMoE-200m-2x16"
+tokenizer = AutoTokenizer.from_pretrained(model_id)
+model = AutoModelForCausalLM.from_pretrained(model_id)
+
+input_text = "Wikipedia is a free"
+inputs = tokenizer(input_text, return_tensors="pt")
+
+outputs = model.generate(**inputs, max_new_tokens=50)
+print(tokenizer.decode(outputs[0], skip_special_tokens=True))
+```

config.json

@@ -0,0 +1,31 @@
+{
+  "architectures": [
+    "MixtralForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "head_dim": null,
+  "hidden_act": "silu",
+  "hidden_size": 384,
+  "initializer_range": 0.02,
+  "intermediate_size": 768,
+  "max_position_embeddings": 1024,
+  "model_type": "mixtral",
+  "num_attention_heads": 8,
+  "num_experts_per_tok": 2,
+  "num_hidden_layers": 12,
+  "num_key_value_heads": 4,
+  "num_local_experts": 16,
+  "output_router_logits": true,
+  "rms_norm_eps": 1e-06,
+  "rope_theta": 1000000.0,
+  "router_aux_loss_coef": 0.02,
+  "router_jitter_noise": 0.0,
+  "sliding_window": 1024,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float32",
+  "transformers_version": "4.55.0",
+  "use_cache": false,
+  "vocab_size": 32000
+}

generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "transformers_version": "4.55.0",
+  "use_cache": false
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fd4680b9cd074de7d3fdd93f57dc1540ffb2557efadf14a0d2d5db41d997684d
+size 799430304

optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a23426e6d3f65b63c5e96957096a72ffa6f9347357ec2c99ff033e839446a7e9
+size 1599258775

rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:068fbd993087219c15b8c0baa13fc39644a4dcdfe92d8be3fa6434deece90371
+size 14244

scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:68d81983c6ddcc582eb27406738b0c1b1b05065e9b9f1da6cd2006446cdf81f5
+size 1064

special_tokens_map.json

@@ -0,0 +1,24 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "</s>",
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,44 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "add_prefix_space": null,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [],
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "extra_special_tokens": {},
+  "legacy": false,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "</s>",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:905a4d16e5da3d49012a41c10e9b80993fd0bcaff9df7dc242e1d8454ea1877e
+size 5368