初始化项目，由ModelHub XC社区提供模型

Model: maywell/Synatra-Mixtral-8x7B
Source: Original Platform

.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,82 @@
+---
+license: apache-2.0
+language:
+- ko
+- en
+tags:
+- moe
+---
+
+# **Synatra-Mixtral-8x7B**
+<img src="./Synatra-Mixtral.png" alt="Synatra-Mixtral-8x7B" width="512"/>
+
+
+**Synatra-Mixtral-8x7B** is a fine-tuned version of the Mixtral-8x7B-Instruct-v0.1 model using **Korean** datasets.
+
+This model features overwhelmingly superior comprehension and inference capabilities and is licensed under apache-2.0.
+
+# **Join Our Discord**
+
+[Server Link](https://discord.gg/MrBt3PXdXc)
+
+# **License**
+
+**OPEN**, Apache-2.0.
+
+# **Model Details**
+
+**Base Model**  
+[mistralai/Mixtral-8x7B-Instruct-v0.1](https://huggingface.co/mistralai/Mixtral-8x7B-Instruct-v0.1)  
+
+**Trained On**  
+A100 80GB * 6
+
+**Instruction format**
+
+It follows **Alpaca** format.
+```
+Below is an instruction that describes a task. Write a response that appropriately completes the request.
+
+### Instruction:
+{input}
+
+### Response:
+{output}
+```
+
+# **Model Benchmark**
+TBD
+
+# **Implementation Code**
+
+```python
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+device = "cuda" # the device to load the model onto
+
+model = AutoModelForCausalLM.from_pretrained("maywell/Synatra-Mixtral-8x7B")
+tokenizer = AutoTokenizer.from_pretrained("maywell/Synatra-Mixtral-8x7B")
+
+messages = [
+    {"role": "user", "content": "아인슈타인의 상대성이론에 대해서 자세히 설명해줘."},
+]
+
+encodeds = tokenizer.apply_chat_template(messages, return_tensors="pt")
+
+model_inputs = encodeds.to(device)
+model.to(device)
+
+generated_ids = model.generate(model_inputs, max_new_tokens=1000, do_sample=True)
+decoded = tokenizer.batch_decode(generated_ids)
+print(decoded[0])
+```
+
+# **Author's Message**
+
+This model's training got sponsered by no one but support from people around Earth.
+
+[Support Me](https://www.buymeacoffee.com/mwell)
+
+Contact Me on Discord - **is.maywell**
+
+Follow me on twitter: https://twitter.com/stablefluffy

config.json

@@ -0,0 +1,30 @@
+{
+  "_name_or_path": "maywell/Synatra-Mixtral-8x7B",
+  "architectures": [
+    "MixtralForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 32768,
+  "model_type": "mixtral",
+  "num_attention_heads": 32,
+  "num_experts_per_tok": 2,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "num_local_experts": 8,
+  "output_router_logits": true,
+  "rms_norm_eps": 1e-05,
+  "rope_theta": 1000000.0,
+  "router_aux_loss_coef": 0.02,
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.37.0.dev0",
+  "use_cache": false,
+  "vocab_size": 32000
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "transformers_version": "4.37.0.dev0"
+}

pytorch_model-00001-of-00019.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:abbb9434f6f993dcb3353c8a459e44a9c6580ec98d214eada5a1ffc3057274f1
+size 4892821505

pytorch_model-00002-of-00019.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:196d64071d1d59f729381f96cdbe3f21eb5ac72a30e2e3cd521a3116c4aa3bee
+size 4983016829

pytorch_model-00003-of-00019.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5ed1a3a29ec23bb08012bfda8c8a194ed4f4606290976595e78e1e2e5ff06f61
+size 4983016913

pytorch_model-00004-of-00019.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:31441768234a6d900dc3dce897d0ad2c6601a9205123f3a55b28a97b4209e77e
+size 4899046399

pytorch_model-00005-of-00019.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:891795cfb3f8a334d08f760bfde3d9ae4695daa3bc1c94e32b0234f90b0df560
+size 4983016865

pytorch_model-00006-of-00019.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:df5adf9034c66f6229163fed006fb9731a796eb8cb5748257fa6dfb2151f9388
+size 4983016893

pytorch_model-00007-of-00019.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:760cc2135776f4e72d6e2f3283168158f734a920778192d6d5696eeaf322f641
+size 4899046463

pytorch_model-00008-of-00019.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7540562870e40a4b3f5a8acf1a511adf2fecc4fb44732084cb1a2babdc7b084b
+size 4983016953

pytorch_model-00009-of-00019.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bb5040aded82219ffa9e78f0f25ddd1047166d14417cc17ba891b7a2513cc530
+size 4983016893

pytorch_model-00010-of-00019.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8582bff83ef1639bf8fe82bf2f0b91d1613e98a95c7411eabe949e0b033ed251
+size 4899046463

pytorch_model-00011-of-00019.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:85e67b8b57c5b309ced69a1d438abf4b4f500b648925aaaf3bd2b37897b80323
+size 4983016917

pytorch_model-00012-of-00019.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4b82dc2e51e6519005d67e1fd467e8b40ae26a31fa0320b56fc5c126f3fca408
+size 4983016917

pytorch_model-00013-of-00019.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:82604cb32a6a31e202078b1912bc882f368b87b0c50b3734b4fdd88078b77f19
+size 4983016893

pytorch_model-00014-of-00019.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b65079309a0163f64d1761b597e920ff8cc886a9b31b98ade1450099249c1f4
+size 4899046463

pytorch_model-00015-of-00019.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:45e6e9d8a1377d98e1fd472d0bdb3e4067307df5acde32dccadb32d674464b0a
+size 4983016953

pytorch_model-00016-of-00019.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:30a0c45e9cc6f0f34b75927c4cce382b8a6c31e266e282f2c43670b83f0ddd6a
+size 4983016893

pytorch_model-00017-of-00019.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:71e52c26fe1158c18ae7db159f277faa2560b7f2f5529767112bbdf00c62b0c3
+size 4899046463

pytorch_model-00018-of-00019.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:63309e951834fb5ffbda2ffd576336d4311dc4287f2f915189e147d5c678b50f
+size 4983016929

pytorch_model-00019-of-00019.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:89ae1fc23bbbb478d3c04f4fcaff63b4f60e65a3792c8b854080e9aba898afa6
+size 4221689255

special_tokens_map.json

@@ -0,0 +1,24 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "</s>",
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,45 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [],
+  "bos_token": "<s>",
+  "chat_template": "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n\n{% for message in messages %}{% if message['role'] == 'user' %}### Instruction:\n{{ message['content']|trim -}}{% if not loop.last %}{% endif %}\n{% elif message['role'] == 'assistant' %}### Response:\n{{ message['content']|trim -}}{% if not loop.last %}{% endif %}\n{% elif message['role'] == 'system' %}{{ message['content']|trim -}}{% if not loop.last %}{% endif %}\n{% endif %}\n{% endfor %}\n{% if add_generation_prompt and messages[-1]['role'] != 'assistant' %}\n### Response:\n{% endif %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "legacy": true,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "</s>",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizer",
+  "trust_remote_code": true,
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false,
+  "use_fast": true
+}