初始化项目，由ModelHub XC社区提供模型

Model: vilm/Mixsmol-4x400M-v0.1-epoch1
Source: Original Platform

.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,70 @@
+---
+license: apache-2.0
+widget:
+- text: My name is El Microondas the Wise, and
+  example_title: El Microondas
+- text: Kennesaw State University is a public
+  example_title: Kennesaw State University
+- text: Bungie Studios is an American video game developer. They are most famous for
+    developing the award winning Halo series of video games. They also made Destiny.
+    The studio was founded
+  example_title: Bungie
+- text: The Mona Lisa is a world-renowned painting created by
+  example_title: Mona Lisa
+- text: The Harry Potter series, written by J.K. Rowling, begins with the book titled
+  example_title: Harry Potter Series
+- text: 'Question: I have cities, but no houses. I have mountains, but no trees. I
+    have water, but no fish. What am I?
+
+    Answer:'
+  example_title: Riddle
+- text: The process of photosynthesis involves the conversion of
+  example_title: Photosynthesis
+- text: Jane went to the store to buy some groceries. She picked up apples, oranges,
+    and a loaf of bread. When she got home, she realized she forgot
+  example_title: Story Continuation
+- text: 'Problem 2: If a train leaves Station A at 9:00 AM and travels at 60 mph,
+    and another train leaves Station B at 10:00 AM and travels at 80 mph, when will
+    they meet if the distance between the stations is 300 miles?
+
+    To determine'
+  example_title: Math Problem
+- text: In the context of computer programming, an algorithm is
+  example_title: Algorithm Definition
+---
+# Mixsmol-4x400M-v0.1 by Ontocord
+This is the first checkpoint (Epoch 1) of Mixsmol-4x400M-v0.1
+Note that this is an experimental in data mixing. Therefore, we only trained the model on 50B tokens (95% English and 5% Vietnamese) to test the following:
+- Reasoining capabilities through high-quality synthetic textbooks data pretraining 
+- Crosslingual understanding through machine translation and multilingual + multiple tasks pretraining
+
+After verifying our hypothesis with this run, we will schedule a second run on bigger data and compute for it to achieve its maximum capability.
+
+## Data
+- Synthetic Textbooks: 8M samples
+- RefinedWeb: 1M samples
+- RedPajama-v2: 500K samples
+- MathPile: Everything
+- ThePile: MiniPile Subset
+- GoodWiki
+- The Stack Smol XL
+- The Vault: train_small split
+- Instruction Pretraining: 250k samples
+
+|    Tasks    |Version|Filter|n-shot| Metric |Value |   |Stderr|
+|-------------|-------|------|-----:|--------|-----:|---|-----:|
+|arc_challenge|Yaml   |none  |    25|acc     |0.1937|±  |0.0115|
+|             |       |none  |    25|acc_norm|0.2329|±  |0.0124|
+|hellaswag|Yaml   |none  |    10|acc     |0.2856|±  |0.0045|
+|         |       |none  |    10|acc_norm|0.3090|±  |0.0046|
+|mmlu              |N/A    |none  |     0|acc   |0.2536|±  |0.0483|
+| - humanities     |N/A    |none  |     5|acc   |0.2408|±  |0.0341|
+| - other          |N/A    |none  |     5|acc   |0.2475|±  |0.0443|
+| - social_sciences|N/A    |none  |     5|acc   |0.2567|±  |0.0456|
+| - stem           |N/A    |none  |     5|acc   |0.2756|±  |0.0653|
+|truthfulqa_mc2|Yaml   |none  |     0|acc   |0.3909|±  |0.0148|
+|winogrande|Yaml   |none  |     5|acc   |0.5107|±  | 0.014|
+|gsm8k|Yaml   |get-answer|     5|exact_match|    0|±  |     0|
+
+## Contribution
+This work is a shared contribution between **Ontocord, BEE-spoke-data and VILM**

config.json

@@ -0,0 +1,30 @@
+{
+  "_name_or_path": "vilm/Mixsmol-4x400M-v0.1-epoch1",
+  "architectures": [
+    "MixtralForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "hidden_act": "silu",
+  "hidden_size": 1536,
+  "initializer_range": 0.02,
+  "intermediate_size": 7168,
+  "max_position_embeddings": 4096,
+  "model_type": "mixtral",
+  "num_attention_heads": 32,
+  "num_experts_per_tok": 2,
+  "num_hidden_layers": 12,
+  "num_key_value_heads": 8,
+  "num_local_experts": 4,
+  "output_router_logits": false,
+  "rms_norm_eps": 1e-05,
+  "rope_theta": 1000000.0,
+  "router_aux_loss_coef": 0.02,
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.37.1",
+  "use_cache": false,
+  "vocab_size": 38365
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:20a0252095a218aa39e1cbb789223895f7d31c1134d0929beda8e770e8d888fa
+size 3548419176

special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e792a804bbfc19a96b61b87109b8f2b0b7c92830025f285b402ba27c0c309c6f
+size 596883

tokenizer_config.json

@@ -0,0 +1,44 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "legacy": true,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<unk>",
+  "padding_side": "right",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "split_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false,
+  "use_fast": true
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4d1d8c93bb8e6c596141a86db73de87b73476dc41cae2f02be2d10855f5314b2
+size 6712