初始化项目，由ModelHub XC社区提供模型

Model: Cartinoe5930/MoE-Merging
Source: Original Platform

.ipynb_checkpoints/config-checkpoint.json

@@ -0,0 +1,30 @@
+{
+  "_name_or_path": "mistralai/Mistral-7B-Instruct-v0.2",
+  "architectures": [
+    "MixtralForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 32768,
+  "model_type": "mixtral",
+  "num_attention_heads": 32,
+  "num_experts_per_tok": 2,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "num_local_experts": 4,
+  "output_router_logits": false,
+  "rms_norm_eps": 1e-05,
+  "rope_theta": 1000000.0,
+  "router_aux_loss_coef": 0.001,
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float16",
+  "transformers_version": "4.37.0",
+  "use_cache": true,
+  "vocab_size": 32000
+}

.ipynb_checkpoints/mergekit_moe_config-checkpoint.yml

@@ -0,0 +1,31 @@
+base_model: mistralai/Mistral-7B-Instruct-v0.2
+dtype: float16
+experts:
+  - source_model: mistralai/Mistral-7B-Instruct-v0.2
+    positive_prompts:
+    - "Provide clear and concise answers"
+    - "Understand the context of the question"
+    - "Give detailed explanations"
+    - "Understand and respond to questions"
+    - "Answer a wide range of questions"
+  - source_model: Open-Orca/Mistral-7B-OpenOrca
+    positive_prompts:
+    - "Can you answer questions"
+    - "Provide a detailed explanation"
+    - "Summarize the main points"
+    - "How does this information relate"
+    - "Tell me more about"
+  - source_model: openchat/openchat-3.5-0106
+    positive_prompts:
+    - "Assist with general conversation"
+    - "Provide information on"
+    - "Generate creative and engaging content"
+    - "Help with problem-solving and critical thinking"
+    - "Provide context and background information on various subjects"
+  - source_model: WizardLM/WizardMath-7B-V1.1
+    positive_prompts:
+    - "Solve this math problem"
+    - "Explain the solution process"
+    - "Provide a step-by-step breakdown"
+    - "Apply this mathematical theorem"
+    - "Simplify this complex math expression"