meraGPT/mera-mix-4x7B
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

初始化项目，由ModelHub XC社区提供模型

Browse Source 
Model: meraGPT/mera-mix-4x7B
Source: Original Platform
This commit is contained in:
ModelHub XC
2026-08-17 23:37:37 +08:00
commit 1071f8195b
18 changed files with 91595 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
.gitattributes vendored Normal file
View File

@@ -0,0 +1,35 @@
*.7z filter=lfs diff=lfs merge=lfs -text
*.arrow filter=lfs diff=lfs merge=lfs -text
*.bin filter=lfs diff=lfs merge=lfs -text
*.bz2 filter=lfs diff=lfs merge=lfs -text
*.ckpt filter=lfs diff=lfs merge=lfs -text
*.ftz filter=lfs diff=lfs merge=lfs -text
*.gz filter=lfs diff=lfs merge=lfs -text
*.h5 filter=lfs diff=lfs merge=lfs -text
*.joblib filter=lfs diff=lfs merge=lfs -text
*.lfs.* filter=lfs diff=lfs merge=lfs -text
*.mlmodel filter=lfs diff=lfs merge=lfs -text
*.model filter=lfs diff=lfs merge=lfs -text
*.msgpack filter=lfs diff=lfs merge=lfs -text
*.npy filter=lfs diff=lfs merge=lfs -text
*.npz filter=lfs diff=lfs merge=lfs -text
*.onnx filter=lfs diff=lfs merge=lfs -text
*.ot filter=lfs diff=lfs merge=lfs -text
*.parquet filter=lfs diff=lfs merge=lfs -text
*.pb filter=lfs diff=lfs merge=lfs -text
*.pickle filter=lfs diff=lfs merge=lfs -text
*.pkl filter=lfs diff=lfs merge=lfs -text
*.pt filter=lfs diff=lfs merge=lfs -text
*.pth filter=lfs diff=lfs merge=lfs -text
*.rar filter=lfs diff=lfs merge=lfs -text
*.safetensors filter=lfs diff=lfs merge=lfs -text
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
*.tar.* filter=lfs diff=lfs merge=lfs -text
*.tar filter=lfs diff=lfs merge=lfs -text
*.tflite filter=lfs diff=lfs merge=lfs -text
*.tgz filter=lfs diff=lfs merge=lfs -text
*.wasm filter=lfs diff=lfs merge=lfs -text
*.xz filter=lfs diff=lfs merge=lfs -text
*.zip filter=lfs diff=lfs merge=lfs -text
*.zst filter=lfs diff=lfs merge=lfs -text
*tfevents* filter=lfs diff=lfs merge=lfs -text

5
.ipynb_checkpoints/added_tokens-checkpoint.json Normal file
View File

@@ -0,0 +1,5 @@
{
"<sep>": 32002,
"<|end_of_turn|>": 32000,
"<|pad_0|>": 32001
}

30
.ipynb_checkpoints/config-checkpoint.json Normal file
View File

@@ -0,0 +1,30 @@
{
"_name_or_path": "berkeley-nest/Starling-LM-7B-alpha",
"architectures": [
"MixtralForCausalLM"
],
"attention_dropout": 0.0,
"bos_token_id": 1,
"eos_token_id": 32000,
"hidden_act": "silu",
"hidden_size": 4096,
"initializer_range": 0.02,
"intermediate_size": 14336,
"max_position_embeddings": 8192,
"model_type": "mixtral",
"num_attention_heads": 32,
"num_experts_per_tok": 1,
"num_hidden_layers": 32,
"num_key_value_heads": 8,
"num_local_experts": 4,
"output_router_logits": false,
"rms_norm_eps": 1e-05,
"rope_theta": 10000.0,
"router_aux_loss_coef": 0.001,
"sliding_window": null,
"tie_word_embeddings": false,
"torch_dtype": "bfloat16",
"transformers_version": "4.39.3",
"use_cache": true,
"vocab_size": 32002
}

71
.ipynb_checkpoints/mergekit_moe_config-checkpoint.yml Normal file
View File

@@ -0,0 +1,71 @@
base_model: chihoonlee10/T3Q-EN-DPO-Mistral-7B
gate_mode: cheap_embed # one of "hidden", "cheap_embed", or "random"
dtype: bfloat16 # output dtype (float32, float16, or bfloat16)
## (optional)
experts_per_token: 2
experts:
- source_model: Kukedlc/Jupiter-k-7B-slerp
positive_prompts:
- "This puzzle involves identifying a repeating pattern. Can you analyze the examples and complete the missing element?"
- "These grids seem to follow a specific visual pattern. Can you analyze the examples and apply the pattern to solve the puzzle?"
- "Look for a consistent rule in how the elements are arranged. Can you use that rule to predict the missing element?"
- "This task requires applying logical deduction. Can you analyze the information and choose the answer that follows logically?"
- "Reason through the cause-and-effect relationships in the examples. Can you use that logic to solve the puzzle?"
- "Analyze the relationship between the input and output in the examples. Can you apply that relationship to solve the new problem?"
- "This puzzle requires modifying the image based on a specific rule. Can you analyze the changes in the examples and apply them to solve the puzzle?"
- "Focus on the visual changes demonstrated in the training examples. Can you replicate those changes to solve the new problem?"
- "This task involves manipulating shapes or colors based on a pattern. Can you analyze the examples and apply the pattern to the new image?"
- "These puzzles involve manipulating numbers according to a specific rule. Can you analyze the pattern and solve the missing number?"
- "Focus on the mathematical operations demonstrated in the examples. Can you apply those operations to solve the new equation?"
- "Look for relationships between the numbers in the training examples. Can you use that relationship to predict the missing number?"
- "This task requires understanding the arrangement of objects in space. Can you analyze the movement patterns in the examples and predict the next step?"
- "Focus on the spatial relationships between elements in the grids. Can you replicate those relationships to solve the new puzzle?"
- "Analyze the rotation, reflection, or translation demonstrated in the examples. Can you apply that manipulation to solve the new problem?"
- source_model: InferenceIllusionist/Excalibur-7b-DPO
positive_prompts:
- "This passage contains factual information. Can you summarize the key details about [topic]?"
- "Based on the information provided, what can you tell me about [entity]?"
- "Is the following statement true or false according to the passage: [statement]?"
- "What caused [event] to happen in the passage?"
- "Why did [character] take the action of [action]?"
- "If [condition] were true, what would likely happen next?"
- "How are [entity A] and [entity B] similar/different?"
- "Which option, [A] or [B], is more likely based on the information provided?"
- "Rank the following options ([list]) based on [criteria] according to the passage."
- "Does the following statement logically follow from the information provided: [statement]?"
- "Identify any inconsistencies or contradictions in the passage."
- "Can you draw a logical conclusion based on the evidence presented?"
- "What can be inferred about [concept] based on the information provided?"
- "What is the underlying meaning or implication of the author's statement?"
- "Can you fill in the blanks with the most likely word(s) based on the context?"
- "What are some potential consequences of [event]?"
- "Can you generate creative solutions to the problem presented in the passage?"
- "Based on the information provided, propose a course of action for [character]."
- source_model: yam-peleg/Experiment21-7B
positive_prompts:
- "Be truthful and objective in your response. Avoid speculation or making claims that cannot be verified."
- "Focus on providing factual information based on the evidence presented in the source material."
- "If you are unsure about something, it's okay to say 'I don't know' or 'I can't find information to support that claim'."
- "Be aware of potential biases in the source material and strive to present a neutral perspective."
- "If a source seems biased, identify the bias and consider alternative viewpoints."
- "Avoid using language that promotes stereotypes or prejudices."
- "Cite your sources when referencing information from external materials."
- "Acknowledge the limitations of your knowledge and the potential for different interpretations."
- "Be transparent about your confidence level in your answer."
- "Explain your reasoning process and how you arrived at your answer."
- "Provide evidence to support your claims whenever possible."
- "If there are multiple perspectives on an issue, present them fairly and objectively."
- source_model: senseable/WestLake-7B-v2
positive_prompts:
- "This sentence contains a pronoun ('he' or 'she'). Pay close attention to the context to determine who the pronoun refers to."
- "Identify the two potential referents for the pronoun ('he' or 'she') in this sentence. Analyze the context to choose the correct one."
- "This scenario describes two individuals. Use the information provided to understand who the pronoun refers to in the sentence."
- "Focus on the actions described in the sentence and the roles of the individuals involved. This will help determine the pronoun referent."
- "Analyze the relationship between the individuals mentioned in the sentence. The pronoun likely refers to the one performing the action."
- "Consider the animacy of the potential referents. Pronouns typically refer to animate beings (people or animals) in the context."
- "Don't rely solely on the pronoun itself. Utilize the entire sentence and surrounding context to understand its meaning."
- "Look for clues in the sentence that indicate who the pronoun refers to. This could include gender, possession, or actions described."
- "Imagine the scenario described in the sentence. Visualizing the situation can help you identify the intended referent."
- "Evaluate the plausibility of each potential referent for the pronoun. Choose the one that makes the most logical sense in the context."
- "Think about the actions described and the roles of the individuals involved. Does it make more sense for one or the other to perform the action?"
- "Consider the world knowledge you possess. Does the sentence describe a situation where one referent is more likely than the other?"

133
README.md Normal file
View File

@@ -0,0 +1,133 @@
---
license: apache-2.0
model-index:
- name: mera-mix-4x7B
results:
- task:
type: text-generation
name: Text Generation
dataset:
name: AI2 Reasoning Challenge (25-Shot)
type: ai2_arc
config: ARC-Challenge
split: test
args:
num_few_shot: 25
metrics:
- type: acc_norm
value: 72.95
name: normalized accuracy
source:
url: https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard?query=meraGPT/mera-mix-4x7B
name: Open LLM Leaderboard
- task:
type: text-generation
name: Text Generation
dataset:
name: HellaSwag (10-Shot)
type: hellaswag
split: validation
args:
num_few_shot: 10
metrics:
- type: acc_norm
value: 89.17
name: normalized accuracy
source:
url: https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard?query=meraGPT/mera-mix-4x7B
name: Open LLM Leaderboard
- task:
type: text-generation
name: Text Generation
dataset:
name: MMLU (5-Shot)
type: cais/mmlu
config: all
split: test
args:
num_few_shot: 5
metrics:
- type: acc
value: 64.44
name: accuracy
source:
url: https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard?query=meraGPT/mera-mix-4x7B
name: Open LLM Leaderboard
- task:
type: text-generation
name: Text Generation
dataset:
name: TruthfulQA (0-shot)
type: truthful_qa
config: multiple_choice
split: validation
args:
num_few_shot: 0
metrics:
- type: mc2
value: 77.17
source:
url: https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard?query=meraGPT/mera-mix-4x7B
name: Open LLM Leaderboard
- task:
type: text-generation
name: Text Generation
dataset:
name: Winogrande (5-shot)
type: winogrande
config: winogrande_xl
split: validation
args:
num_few_shot: 5
metrics:
- type: acc
value: 85.64
name: accuracy
source:
url: https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard?query=meraGPT/mera-mix-4x7B
name: Open LLM Leaderboard
- task:
type: text-generation
name: Text Generation
dataset:
name: GSM8k (5-shot)
type: gsm8k
config: main
split: test
args:
num_few_shot: 5
metrics:
- type: acc
value: 66.11
name: accuracy
source:
url: https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard?query=meraGPT/mera-mix-4x7B
name: Open LLM Leaderboard
---
# Model mera-mix-4x7B
This is a mixture of experts (MoE) model that is half as large (4 experts instead of 8) as the [Mixtral-8x7B](https://huggingface.co/mistralai/Mixtral-8x7B-Instruct-v0.1)
while been comparable to it across different benchmarks. You can use it as a drop in replacement for your Mixtral-8x7B and get much faster inference.
mera-mix-4x7B achieves the score of 75.91 on the OpenLLM Eval and compares well with 72.7 by Mixtral-8x7B and 74.46 by Mixtral-8x22B.
You can try the model with the [Mera Mixture Chat](https://huggingface.co/spaces/meraGPT/mera-mixture-chat).
In addition, to the official Open LLM Leaderboard, the results on OpenLLM Eval have been validated by [others as well (76.59)](https://github.com/saucam/model_evals/tree/main?tab=readme-ov-file#model-eval-results).
Our own initial eval is available [here (76.37)](https://gist.github.com/codelion/78f88333230801c9bbaa6fc22078d820).
# [Open LLM Leaderboard Evaluation Results](https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard)
Detailed results can be found [here](https://huggingface.co/datasets/open-llm-leaderboard/details_meraGPT__mera-mix-4x7B)
| Metric |Value|
|---------------------------------|----:|
|Avg. |75.91|
|AI2 Reasoning Challenge (25-Shot)|72.95|
|HellaSwag (10-Shot) |89.17|
|MMLU (5-Shot) |64.44|
|TruthfulQA (0-shot) |77.17|
|Winogrande (5-shot) |85.64|
|GSM8k (5-shot) |66.11|

5
added_tokens.json Normal file
View File

@@ -0,0 +1,5 @@
{
"<sep>": 32002,
"<|end_of_turn|>": 32000,
"<|pad_0|>": 32001
}

30
config.json Normal file
View File

@@ -0,0 +1,30 @@
{
"_name_or_path": "chihoonlee10/T3Q-EN-DPO-Mistral-7B",
"architectures": [
"MixtralForCausalLM"
],
"attention_dropout": 0.0,
"bos_token_id": 1,
"eos_token_id": 2,
"hidden_act": "silu",
"hidden_size": 4096,
"initializer_range": 0.02,
"intermediate_size": 14336,
"max_position_embeddings": 32768,
"model_type": "mixtral",
"num_attention_heads": 32,
"num_experts_per_tok": 2,
"num_hidden_layers": 32,
"num_key_value_heads": 8,
"num_local_experts": 4,
"output_router_logits": false,
"rms_norm_eps": 1e-05,
"rope_theta": 10000.0,
"router_aux_loss_coef": 0.001,
"sliding_window": null,
"tie_word_embeddings": false,
"torch_dtype": "bfloat16",
"transformers_version": "4.39.3",
"use_cache": true,
"vocab_size": 32000
}

71
mergekit_moe_config.yml Normal file
View File

@@ -0,0 +1,71 @@
base_model: chihoonlee10/T3Q-EN-DPO-Mistral-7B
gate_mode: hidden # one of "hidden", "cheap_embed", or "random"
dtype: bfloat16 # output dtype (float32, float16, or bfloat16)
## (optional)
experts_per_token: 2
experts:
- source_model: Kukedlc/Jupiter-k-7B-slerp
positive_prompts:
- "This puzzle involves identifying a repeating pattern. Can you analyze the examples and complete the missing element?"
- "These grids seem to follow a specific visual pattern. Can you analyze the examples and apply the pattern to solve the puzzle?"
- "Look for a consistent rule in how the elements are arranged. Can you use that rule to predict the missing element?"
- "This task requires applying logical deduction. Can you analyze the information and choose the answer that follows logically?"
- "Reason through the cause-and-effect relationships in the examples. Can you use that logic to solve the puzzle?"
- "Analyze the relationship between the input and output in the examples. Can you apply that relationship to solve the new problem?"
- "This puzzle requires modifying the image based on a specific rule. Can you analyze the changes in the examples and apply them to solve the puzzle?"
- "Focus on the visual changes demonstrated in the training examples. Can you replicate those changes to solve the new problem?"
- "This task involves manipulating shapes or colors based on a pattern. Can you analyze the examples and apply the pattern to the new image?"
- "These puzzles involve manipulating numbers according to a specific rule. Can you analyze the pattern and solve the missing number?"
- "Focus on the mathematical operations demonstrated in the examples. Can you apply those operations to solve the new equation?"
- "Look for relationships between the numbers in the training examples. Can you use that relationship to predict the missing number?"
- "This task requires understanding the arrangement of objects in space. Can you analyze the movement patterns in the examples and predict the next step?"
- "Focus on the spatial relationships between elements in the grids. Can you replicate those relationships to solve the new puzzle?"
- "Analyze the rotation, reflection, or translation demonstrated in the examples. Can you apply that manipulation to solve the new problem?"
- source_model: InferenceIllusionist/Excalibur-7b-DPO
positive_prompts:
- "This passage contains factual information. Can you summarize the key details about [topic]?"
- "Based on the information provided, what can you tell me about [entity]?"
- "Is the following statement true or false according to the passage: [statement]?"
- "What caused [event] to happen in the passage?"
- "Why did [character] take the action of [action]?"
- "If [condition] were true, what would likely happen next?"
- "How are [entity A] and [entity B] similar/different?"
- "Which option, [A] or [B], is more likely based on the information provided?"
- "Rank the following options ([list]) based on [criteria] according to the passage."
- "Does the following statement logically follow from the information provided: [statement]?"
- "Identify any inconsistencies or contradictions in the passage."
- "Can you draw a logical conclusion based on the evidence presented?"
- "What can be inferred about [concept] based on the information provided?"
- "What is the underlying meaning or implication of the author's statement?"
- "Can you fill in the blanks with the most likely word(s) based on the context?"
- "What are some potential consequences of [event]?"
- "Can you generate creative solutions to the problem presented in the passage?"
- "Based on the information provided, propose a course of action for [character]."
- source_model: yam-peleg/Experiment21-7B
positive_prompts:
- "Be truthful and objective in your response. Avoid speculation or making claims that cannot be verified."
- "Focus on providing factual information based on the evidence presented in the source material."
- "If you are unsure about something, it's okay to say 'I don't know' or 'I can't find information to support that claim'."
- "Be aware of potential biases in the source material and strive to present a neutral perspective."
- "If a source seems biased, identify the bias and consider alternative viewpoints."
- "Avoid using language that promotes stereotypes or prejudices."
- "Cite your sources when referencing information from external materials."
- "Acknowledge the limitations of your knowledge and the potential for different interpretations."
- "Be transparent about your confidence level in your answer."
- "Explain your reasoning process and how you arrived at your answer."
- "Provide evidence to support your claims whenever possible."
- "If there are multiple perspectives on an issue, present them fairly and objectively."
- source_model: senseable/WestLake-7B-v2
positive_prompts:
- "This sentence contains a pronoun ('he' or 'she'). Pay close attention to the context to determine who the pronoun refers to."
- "Identify the two potential referents for the pronoun ('he' or 'she') in this sentence. Analyze the context to choose the correct one."
- "This scenario describes two individuals. Use the information provided to understand who the pronoun refers to in the sentence."
- "Focus on the actions described in the sentence and the roles of the individuals involved. This will help determine the pronoun referent."
- "Analyze the relationship between the individuals mentioned in the sentence. The pronoun likely refers to the one performing the action."
- "Consider the animacy of the potential referents. Pronouns typically refer to animate beings (people or animals) in the context."
- "Don't rely solely on the pronoun itself. Utilize the entire sentence and surrounding context to understand its meaning."
- "Look for clues in the sentence that indicate who the pronoun refers to. This could include gender, possession, or actions described."
- "Imagine the scenario described in the sentence. Visualizing the situation can help you identify the intended referent."
- "Evaluate the plausibility of each potential referent for the pronoun. Choose the one that makes the most logical sense in the context."
- "Think about the actions described and the roles of the individuals involved. Does it make more sense for one or the other to perform the action?"
- "Consider the world knowledge you possess. Does the sentence describe a situation where one referent is more likely than the other?"

3
model-00001-of-00005.safetensors Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:b3391957ad1fcacac319678cbc5d2a3871e854c8eb5abedf19d46d8a4c0fba14
size 9919666872

3
model-00002-of-00005.safetensors Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:1f41a8f5ae493bf80ce115a3c436537dae54ed8a09540d80a5103251c432147f
size 9898670488

3
model-00003-of-00005.safetensors Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:2d6695ac15e3bf03106ae33f695710a8d8807d227010123e2e3e5e8759211972
size 9982573720

3
model-00004-of-00005.safetensors Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:e227e74b71acecaf8d2572590f2861f4cd5812738ea3d437ac920c91429624cc
size 9982573720

3
model-00005-of-00005.safetensors Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:cb15c9ac1d085517ea659cb20cebc463b4cf896586bcf9f60456a6c12da0c39c
size 8523972544

1
model.safetensors.index.json Normal file
View File

File diff suppressed because one or more lines are too long

24
special_tokens_map.json Normal file
View File

@@ -0,0 +1,24 @@
{
"bos_token": {
"content": "<s>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
"eos_token": {
"content": "</s>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
"pad_token": "<s>",
"unk_token": {
"content": "<unk>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
}
}

91129
tokenizer.json Normal file
View File

File diff suppressed because it is too large Load Diff

BIN
tokenizer.model (Stored with Git LFS) Normal file
View File

Binary file not shown.

43
tokenizer_config.json Normal file
View File

@@ -0,0 +1,43 @@
{
"add_bos_token": true,
"add_eos_token": false,
"added_tokens_decoder": {
"0": {
"content": "<unk>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"1": {
"content": "<s>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"2": {
"content": "</s>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
}
},
"additional_special_tokens": [],
"bos_token": "<s>",
"clean_up_tokenization_spaces": false,
"eos_token": "</s>",
"legacy": true,
"model_max_length": 32768,
"pad_token": "<s>",
"padding_side": "left",
"sp_model_kwargs": {},
"spaces_between_special_tokens": false,
"tokenizer_class": "LlamaTokenizer",
"unk_token": "<unk>",
"use_default_system_prompt": false
}