初始化项目，由ModelHub XC社区提供模型
Model: Naphula/Meme-Trix-MoE-14B-A8B-v1 Source: Original Platform
This commit is contained in:
ModelHub XC
37
.gitattributes
vendored
Normal file
37
.gitattributes
vendored
Normal file
@@ -0,0 +1,37 @@
|
||||
*.7z filter=lfs diff=lfs merge=lfs -text
|
||||
*.arrow filter=lfs diff=lfs merge=lfs -text
|
||||
*.bin filter=lfs diff=lfs merge=lfs -text
|
||||
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
||||
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
||||
*.ftz filter=lfs diff=lfs merge=lfs -text
|
||||
*.gz filter=lfs diff=lfs merge=lfs -text
|
||||
*.h5 filter=lfs diff=lfs merge=lfs -text
|
||||
*.joblib filter=lfs diff=lfs merge=lfs -text
|
||||
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
||||
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
||||
*.model filter=lfs diff=lfs merge=lfs -text
|
||||
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
||||
*.npy filter=lfs diff=lfs merge=lfs -text
|
||||
*.npz filter=lfs diff=lfs merge=lfs -text
|
||||
*.onnx filter=lfs diff=lfs merge=lfs -text
|
||||
*.ot filter=lfs diff=lfs merge=lfs -text
|
||||
*.parquet filter=lfs diff=lfs merge=lfs -text
|
||||
*.pb filter=lfs diff=lfs merge=lfs -text
|
||||
*.pickle filter=lfs diff=lfs merge=lfs -text
|
||||
*.pkl filter=lfs diff=lfs merge=lfs -text
|
||||
*.pt filter=lfs diff=lfs merge=lfs -text
|
||||
*.pth filter=lfs diff=lfs merge=lfs -text
|
||||
*.rar filter=lfs diff=lfs merge=lfs -text
|
||||
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
||||
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
||||
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
||||
*.tar filter=lfs diff=lfs merge=lfs -text
|
||||
*.tflite filter=lfs diff=lfs merge=lfs -text
|
||||
*.tgz filter=lfs diff=lfs merge=lfs -text
|
||||
*.wasm filter=lfs diff=lfs merge=lfs -text
|
||||
*.xz filter=lfs diff=lfs merge=lfs -text
|
||||
*.zip filter=lfs diff=lfs merge=lfs -text
|
||||
*.zst filter=lfs diff=lfs merge=lfs -text
|
||||
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
||||
Meme-Trix.png filter=lfs diff=lfs merge=lfs -text
|
||||
tokenizer.json filter=lfs diff=lfs merge=lfs -text
|
||||
3
Meme-Trix.png
Normal file
3
Meme-Trix.png
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:474c1a7c5c3b709e9418d728fe872b65f248600fe832456296873930807f9692
|
||||
size 1556255
|
||||
74
README.md
Normal file
74
README.md
Normal file
@@ -0,0 +1,74 @@
|
||||
---
|
||||
license: apache-2.0
|
||||
base_model:
|
||||
- OccultAI/Morpheus-8B-v1
|
||||
- SicariusSicariiStuff/Assistant_Pepe_8B
|
||||
tags:
|
||||
- merge
|
||||
- llama
|
||||
- occult
|
||||
- uncensored
|
||||
- moe
|
||||
datasets:
|
||||
- OccultAI/Morpheus_665
|
||||
- SicariusSicariiStuff/UBW_Tapestries
|
||||
language:
|
||||
- en
|
||||
library_name: transformers
|
||||
widget:
|
||||
- text: "Meme-Trix MoE 14B A8B v1"
|
||||
output:
|
||||
url: https://cdn-uploads.huggingface.co/production/uploads/68e840caa318194c44ec2a04/hEyahuR43IMF5TC4XrHGS.png
|
||||
new_version: Naphula/Meme-Trix-MoE-14B-A8B-v2
|
||||
---
|
||||
|
||||
> [!CAUTION]
|
||||
> <span style="color:red; font-weight:bold">⚠️ Warning:</span> This model can produce narratives and RP that contain violent and graphic erotic content. Adjust your system prompt accordingly, and use **Llama 3** chat template.
|
||||
>
|
||||
|
||||
# Meme-Trix MoE 14B A8B v1
|
||||
|
||||
|
||||
|
||||
A custom built Llama 3.1 8B MoE (Mixture of Experts) merge which combines Morpheus v1 with Assistant Pepe. The merge is suprisingly intelligent, detailed, and based. Scores ~15K at Q0 Bench. Fully uncensored and almost as fast as 8B dense. It also appears to have strong context retrieval ability. Asked for a summary at 16K and it works flawlessly (did not test higher yet).
|
||||
|
||||
If you want to merge custom Llama MoE you can add these scripts to your mergekit environment:
|
||||
- [mergekit-main\mergekit\architecture\moe_defs.py](https://huggingface.co/Naphula/Meme-Trix-MoE-14B-A8B-v1/blob/main/moe_defs.py)
|
||||
- [mergekit-main\mergekit\moe\__init__.py](https://huggingface.co/Naphula/Meme-Trix-MoE-14B-A8B-v1/blob/main/__init__.py)
|
||||
- [mergekit-main\mergekit\moe\llama.py](https://huggingface.co/Naphula/Meme-Trix-MoE-14B-A8B-v1/blob/main/llama.py)
|
||||
|
||||
Then assign the `num_experts_per_tok` in `config.json` (or the `config.yaml`)
|
||||
|
||||
## Recommended Settings
|
||||
**(bolded kobold non-defaults)**
|
||||
- **Temp 1.0**
|
||||
- **TopNSigma 1.25**
|
||||
- **Min-P 0.1**
|
||||
- **Repetition Penalty 1.08**
|
||||
- **Top-P 1.0**
|
||||
- Top-K 100
|
||||
- Top-A 0
|
||||
- Typical Sampling 1
|
||||
- Tail-Free Sampling 1
|
||||
- Presence Penalty 0
|
||||
- Sampler Seed -1
|
||||
- Rp.Range 360
|
||||
- Rp.Slope 0.7
|
||||
- Smoothing Factor 0
|
||||
- Smoothing Curve 1
|
||||
- DynaTemp 0
|
||||
- Mirostat Mode OFF ("2" enhances creativity but also errors)
|
||||
- Mirostat Tau 5
|
||||
- Mirostat Eta 0.1
|
||||
- **DRY Multiplier 0.8**
|
||||
- DRY Base 1.75
|
||||
- DRY A.Len 2
|
||||
- DRY L.Len 320
|
||||
- **XTC Threshold 0.1**
|
||||
- **XTC Probability 0.08** (The "Anti-Cliche" Shield)
|
||||
- **DynaTemp ON** (The "Poor Man's Fading Mirostat")
|
||||
- **Minimum Temperature 0.65**
|
||||
- **Maximum Temperature 1.35**
|
||||
- **Temperature 1.0**
|
||||
- **DynaTemp-Range 0.35**
|
||||
- **DynaTemp-Exponent 1**
|
||||
36
__init__.py
Normal file
36
__init__.py
Normal file
@@ -0,0 +1,36 @@
|
||||
from typing import List
|
||||
|
||||
from mergekit.moe.arch import MoEOutputArchitecture
|
||||
from mergekit.moe.deepseek import DeepseekMoE
|
||||
from mergekit.moe.mixtral import MixtralMoE
|
||||
|
||||
ALL_OUTPUT_ARCHITECTURES: List[MoEOutputArchitecture] = [MixtralMoE(), DeepseekMoE()]
|
||||
|
||||
try:
|
||||
from mergekit.moe.qwen import QwenMoE
|
||||
except ImportError:
|
||||
pass
|
||||
else:
|
||||
ALL_OUTPUT_ARCHITECTURES.append(QwenMoE())
|
||||
|
||||
try:
|
||||
from mergekit.moe.qwen3 import Qwen3MoE
|
||||
except ImportError:
|
||||
pass
|
||||
else:
|
||||
ALL_OUTPUT_ARCHITECTURES.append(Qwen3MoE())
|
||||
|
||||
# --- ADD THIS SECTION START ---
|
||||
try:
|
||||
from mergekit.moe.llama import LlamaMoE
|
||||
except ImportError:
|
||||
# This will trigger if llama.py is missing or has a syntax error
|
||||
pass
|
||||
else:
|
||||
ALL_OUTPUT_ARCHITECTURES.append(LlamaMoE())
|
||||
# --- ADD THIS SECTION END ---
|
||||
|
||||
__all__ = [
|
||||
"ALL_OUTPUT_ARCHITECTURES",
|
||||
"MoEOutputArchitecture",
|
||||
]
|
||||
5
chat_template.jinja
Normal file
5
chat_template.jinja
Normal file
@@ -0,0 +1,5 @@
|
||||
{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% set loop_messages = messages %}{% for message in loop_messages %}{% set content = '<|start_header_id|>' + message['role'] + '<|end_header_id|>
|
||||
|
||||
'+ message['content'] | trim + '<|eot_id|>' %}{% if loop.index0 == 0 %}{% set content = bos_token + content %}{% endif %}{{ content }}{% endfor %}{% if add_generation_prompt %}{{ '<|start_header_id|>assistant<|end_header_id|>
|
||||
|
||||
' }}{% endif %}
|
||||
37
config.json
Normal file
37
config.json
Normal file
@@ -0,0 +1,37 @@
|
||||
{
|
||||
"architectures": [
|
||||
"MixtralForCausalLM"
|
||||
],
|
||||
"attention_bias": false,
|
||||
"attention_dropout": 0.0,
|
||||
"bos_token_id": 128000,
|
||||
"eos_token_id": 128009,
|
||||
"hidden_act": "silu",
|
||||
"hidden_size": 4096,
|
||||
"initializer_range": 0.02,
|
||||
"intermediate_size": 14336,
|
||||
"max_position_embeddings": 1073152,
|
||||
"model_type": "mixtral",
|
||||
"num_attention_heads": 32,
|
||||
"num_experts_per_tok": 2,
|
||||
"num_hidden_layers": 32,
|
||||
"num_key_value_heads": 8,
|
||||
"num_local_experts": 2,
|
||||
"output_router_logits": false,
|
||||
"rms_norm_eps": 1e-05,
|
||||
"rope_scaling": {
|
||||
"factor": 128.0,
|
||||
"high_freq_factor": 4.0,
|
||||
"low_freq_factor": 1.0,
|
||||
"original_max_position_embeddings": 8192,
|
||||
"rope_type": "llama3"
|
||||
},
|
||||
"rope_theta": 500000.0,
|
||||
"router_aux_loss_coef": 0.02,
|
||||
"sliding_window": null,
|
||||
"tie_word_embeddings": false,
|
||||
"torch_dtype": "bfloat16",
|
||||
"transformers_version": "4.45.2",
|
||||
"use_cache": true,
|
||||
"vocab_size": 129024
|
||||
}
|
||||
6
generation_config.json
Normal file
6
generation_config.json
Normal file
@@ -0,0 +1,6 @@
|
||||
{
|
||||
"bos_token_id": 128000,
|
||||
"do_sample": true,
|
||||
"eos_token_id": 128009,
|
||||
"transformers_version": "4.45.2"
|
||||
}
|
||||
59
llama.py
Normal file
59
llama.py
Normal file
@@ -0,0 +1,59 @@
|
||||
import torch
|
||||
import tqdm
|
||||
import transformers
|
||||
from mergekit.moe.arch import MoEOutputArchitecture
|
||||
from mergekit.moe.common import copy_tensor_out, initialize_io, select_dtype
|
||||
from mergekit.moe.config import MoEMergeConfig
|
||||
from mergekit.options import MergeOptions
|
||||
from mergekit.architecture import arch_info_for_config
|
||||
|
||||
class LlamaMoE(MoEOutputArchitecture):
|
||||
def name(self) -> str:
|
||||
return "LlamaMoE"
|
||||
|
||||
def supports_config(self, config: MoEMergeConfig, explain: bool = False, trust_remote_code: bool = False) -> bool:
|
||||
# Ensure the base model is a Llama model
|
||||
model_cfg = config.base_model.config(trust_remote_code=trust_remote_code)
|
||||
if model_cfg.model_type != "llama":
|
||||
if explain:
|
||||
print("LlamaMoE only supports Llama base models")
|
||||
return False
|
||||
return True
|
||||
|
||||
def write_model(self, out_path: str, config: MoEMergeConfig, merge_options: MergeOptions, router_weights: list[torch.Tensor], shared_router_weights=None):
|
||||
base_model = config.base_model
|
||||
base_cfg = base_model.config(trust_remote_code=merge_options.trust_remote_code)
|
||||
|
||||
# 1. Generate the config.json
|
||||
out_cfg = base_cfg.to_dict()
|
||||
# Note: Most Llama MoEs use the Mixtral architecture name for compatibility with loaders
|
||||
out_cfg["architectures"] = ["MixtralForCausalLM"]
|
||||
out_cfg["num_local_experts"] = len(config.experts)
|
||||
out_cfg["num_experts_per_tok"] = config.experts_per_token
|
||||
|
||||
out_dtype = select_dtype(config, base_cfg)
|
||||
|
||||
# 2. Initialize IO
|
||||
loaders, base_loader, writer = initialize_io(config, out_path, merge_options)
|
||||
|
||||
# 3. Map Tensors
|
||||
for weight_info in tqdm.tqdm(arch_info_for_config(base_cfg).all_weights(base_cfg), desc="Weights"):
|
||||
tensor_name = weight_info.name
|
||||
if ".mlp." in tensor_name:
|
||||
for expert_idx, expert in enumerate(config.experts):
|
||||
# Map Llama's gate_proj/up_proj/down_proj to Mixtral's w1/w3/w2
|
||||
expert_name = tensor_name.replace(".mlp.gate_proj", f".block_sparse_moe.experts.{expert_idx}.w1")
|
||||
expert_name = expert_name.replace(".mlp.down_proj", f".block_sparse_moe.experts.{expert_idx}.w2")
|
||||
expert_name = expert_name.replace(".mlp.up_proj", f".block_sparse_moe.experts.{expert_idx}.w3")
|
||||
|
||||
expert_loader = loaders.get(expert.source_model)
|
||||
copy_tensor_out(weight_info, expert_loader, writer, expert=expert, output_name=expert_name, out_dtype=out_dtype)
|
||||
else:
|
||||
# Copy Attention and Norms from base model
|
||||
copy_tensor_out(weight_info, base_loader, writer, out_dtype=out_dtype)
|
||||
|
||||
# 4. Write Router Weights
|
||||
for layer_idx, weight in enumerate(router_weights):
|
||||
writer.save_tensor(f"model.layers.{layer_idx}.block_sparse_moe.gate.weight", weight.to(dtype=out_dtype))
|
||||
|
||||
writer.finalize()
|
||||
12
mergekit_moe_config.yml
Normal file
12
mergekit_moe_config.yml
Normal file
@@ -0,0 +1,12 @@
|
||||
base_model: B:\8B\!models--SicariusSicariiStuff--Assistant_Pepe_8B
|
||||
gate_mode: hidden
|
||||
dtype: bfloat16
|
||||
architecture: LlamaMoE # This must match the name() method in your llama.py class
|
||||
experts:
|
||||
- source_model: B:\8B\!models--SicariusSicariiStuff--Llama-3.1-Nemotron-8B-UltraLong-1M-Instruct_Abliterated\!Morpheus_v1_8B_finetuned_adapter\checkpoint-668\v1
|
||||
positive_prompts:
|
||||
- "Who is Morpheus?"
|
||||
- source_model: B:\8B\!models--SicariusSicariiStuff--Assistant_Pepe_8B
|
||||
positive_prompts:
|
||||
- "Who is Pepe?"
|
||||
# timeout /t 3 /nobreak && mergekit-moe llama_moe.yaml C:/mergekit-main/Llama-3.1-8B-Morpheus-Pepe-MoE --copy-tokenizer --allow-crimes --out-shard-size 5B --trust-remote-code --lazy-unpickle --random-seed 420 --device cuda --load-in-4bit
|
||||
3
model-00001-of-00006.safetensors
Normal file
3
model-00001-of-00006.safetensors
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:f08a9141618ee6ebf7b5567af98be3e66b4e9011a57f3bfe181828b147f26741
|
||||
size 4999708160
|
||||
3
model-00002-of-00006.safetensors
Normal file
3
model-00002-of-00006.safetensors
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:ae0500a0ec8e742abc17151be65c52dc3c03edccbd24b1a8b96a80965c758692
|
||||
size 4932617648
|
||||
3
model-00003-of-00006.safetensors
Normal file
3
model-00003-of-00006.safetensors
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:cc5ae92ee2b893a3d458c32da7e8cbe859e9b6f9a57400184b610405f112ffc3
|
||||
size 4966163504
|
||||
3
model-00004-of-00006.safetensors
Normal file
3
model-00004-of-00006.safetensors
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:132c7e11adfbba33b1f93c30593a65139d95fa9a084d0fa7b13b43acb65c3a39
|
||||
size 4966163504
|
||||
3
model-00005-of-00006.safetensors
Normal file
3
model-00005-of-00006.safetensors
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:deeb04e91d9e324c8d1a96ec878b4120e851067afd2d639694d4c3b6c35415d2
|
||||
size 4932626016
|
||||
3
model-00006-of-00006.safetensors
Normal file
3
model-00006-of-00006.safetensors
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:4aa646586209ae261ed203ffd05907c46068a33693612f6397f48e77cfe9d2e4
|
||||
size 2550692032
|
||||
427
model.safetensors.index.json
Normal file
427
model.safetensors.index.json
Normal file
@@ -0,0 +1,427 @@
|
||||
{
|
||||
"metadata": {
|
||||
"total_size": 27347918848,
|
||||
"mergekit_version": "0.1.4"
|
||||
},
|
||||
"weight_map": {
|
||||
"model.embed_tokens.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.0.input_layernorm.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.0.self_attn.q_proj.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.0.self_attn.k_proj.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.0.self_attn.v_proj.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.0.self_attn.o_proj.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.0.post_attention_layernorm.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.0.block_sparse_moe.experts.0.w3.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.0.block_sparse_moe.experts.1.w3.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.0.block_sparse_moe.experts.0.w1.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.0.block_sparse_moe.experts.1.w1.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.0.block_sparse_moe.experts.0.w2.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.0.block_sparse_moe.experts.1.w2.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.1.input_layernorm.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.1.self_attn.q_proj.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.1.self_attn.k_proj.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.1.self_attn.v_proj.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.1.self_attn.o_proj.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.1.post_attention_layernorm.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.1.block_sparse_moe.experts.0.w3.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.1.block_sparse_moe.experts.1.w3.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.1.block_sparse_moe.experts.0.w1.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.1.block_sparse_moe.experts.1.w1.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.1.block_sparse_moe.experts.0.w2.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.1.block_sparse_moe.experts.1.w2.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.2.input_layernorm.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.2.self_attn.q_proj.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.2.self_attn.k_proj.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.2.self_attn.v_proj.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.2.self_attn.o_proj.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.2.post_attention_layernorm.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.2.block_sparse_moe.experts.0.w3.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.2.block_sparse_moe.experts.1.w3.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.2.block_sparse_moe.experts.0.w1.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.2.block_sparse_moe.experts.1.w1.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.2.block_sparse_moe.experts.0.w2.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.2.block_sparse_moe.experts.1.w2.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.3.input_layernorm.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.3.self_attn.q_proj.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.3.self_attn.k_proj.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.3.self_attn.v_proj.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.3.self_attn.o_proj.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.3.post_attention_layernorm.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.3.block_sparse_moe.experts.0.w3.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.3.block_sparse_moe.experts.1.w3.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.3.block_sparse_moe.experts.0.w1.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.3.block_sparse_moe.experts.1.w1.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.3.block_sparse_moe.experts.0.w2.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.3.block_sparse_moe.experts.1.w2.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.4.input_layernorm.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.4.self_attn.q_proj.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.4.self_attn.k_proj.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.4.self_attn.v_proj.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.4.self_attn.o_proj.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.4.post_attention_layernorm.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.4.block_sparse_moe.experts.0.w3.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.4.block_sparse_moe.experts.1.w3.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.4.block_sparse_moe.experts.0.w1.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.4.block_sparse_moe.experts.1.w1.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.4.block_sparse_moe.experts.0.w2.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.4.block_sparse_moe.experts.1.w2.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.5.input_layernorm.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.5.self_attn.q_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.5.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.5.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.5.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.5.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.5.block_sparse_moe.experts.0.w3.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.5.block_sparse_moe.experts.1.w3.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.5.block_sparse_moe.experts.0.w1.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.5.block_sparse_moe.experts.1.w1.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.5.block_sparse_moe.experts.0.w2.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.5.block_sparse_moe.experts.1.w2.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.6.input_layernorm.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.6.self_attn.q_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.6.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.6.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.6.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.6.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.6.block_sparse_moe.experts.0.w3.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.6.block_sparse_moe.experts.1.w3.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.6.block_sparse_moe.experts.0.w1.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.6.block_sparse_moe.experts.1.w1.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.6.block_sparse_moe.experts.0.w2.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.6.block_sparse_moe.experts.1.w2.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.7.input_layernorm.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.7.self_attn.q_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.7.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.7.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.7.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.7.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.7.block_sparse_moe.experts.0.w3.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.7.block_sparse_moe.experts.1.w3.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.7.block_sparse_moe.experts.0.w1.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.7.block_sparse_moe.experts.1.w1.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.7.block_sparse_moe.experts.0.w2.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.7.block_sparse_moe.experts.1.w2.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.8.input_layernorm.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.8.self_attn.q_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.8.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.8.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.8.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.8.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.8.block_sparse_moe.experts.0.w3.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.8.block_sparse_moe.experts.1.w3.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.8.block_sparse_moe.experts.0.w1.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.8.block_sparse_moe.experts.1.w1.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.8.block_sparse_moe.experts.0.w2.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.8.block_sparse_moe.experts.1.w2.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.9.input_layernorm.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.9.self_attn.q_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.9.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.9.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.9.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.9.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.9.block_sparse_moe.experts.0.w3.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.9.block_sparse_moe.experts.1.w3.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.9.block_sparse_moe.experts.0.w1.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.9.block_sparse_moe.experts.1.w1.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.9.block_sparse_moe.experts.0.w2.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.9.block_sparse_moe.experts.1.w2.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.10.input_layernorm.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.10.self_attn.q_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.10.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.10.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.10.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.10.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.10.block_sparse_moe.experts.0.w3.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.10.block_sparse_moe.experts.1.w3.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.10.block_sparse_moe.experts.0.w1.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.10.block_sparse_moe.experts.1.w1.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.10.block_sparse_moe.experts.0.w2.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.10.block_sparse_moe.experts.1.w2.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.11.input_layernorm.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.11.self_attn.q_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.11.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.11.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.11.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.11.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.11.block_sparse_moe.experts.0.w3.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.11.block_sparse_moe.experts.1.w3.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.11.block_sparse_moe.experts.0.w1.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.11.block_sparse_moe.experts.1.w1.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.11.block_sparse_moe.experts.0.w2.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.11.block_sparse_moe.experts.1.w2.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.12.input_layernorm.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.12.self_attn.q_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.12.self_attn.k_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.12.self_attn.v_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.12.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.12.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.12.block_sparse_moe.experts.0.w3.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.12.block_sparse_moe.experts.1.w3.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.12.block_sparse_moe.experts.0.w1.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.12.block_sparse_moe.experts.1.w1.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.12.block_sparse_moe.experts.0.w2.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.12.block_sparse_moe.experts.1.w2.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.13.input_layernorm.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.13.self_attn.q_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.13.self_attn.k_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.13.self_attn.v_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.13.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.13.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.13.block_sparse_moe.experts.0.w3.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.13.block_sparse_moe.experts.1.w3.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.13.block_sparse_moe.experts.0.w1.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.13.block_sparse_moe.experts.1.w1.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.13.block_sparse_moe.experts.0.w2.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.13.block_sparse_moe.experts.1.w2.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.14.input_layernorm.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.14.self_attn.q_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.14.self_attn.k_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.14.self_attn.v_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.14.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.14.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.14.block_sparse_moe.experts.0.w3.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.14.block_sparse_moe.experts.1.w3.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.14.block_sparse_moe.experts.0.w1.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.14.block_sparse_moe.experts.1.w1.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.14.block_sparse_moe.experts.0.w2.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.14.block_sparse_moe.experts.1.w2.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.15.input_layernorm.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.15.self_attn.q_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.15.self_attn.k_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.15.self_attn.v_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.15.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.15.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.15.block_sparse_moe.experts.0.w3.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.15.block_sparse_moe.experts.1.w3.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.15.block_sparse_moe.experts.0.w1.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.15.block_sparse_moe.experts.1.w1.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.15.block_sparse_moe.experts.0.w2.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.15.block_sparse_moe.experts.1.w2.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.16.input_layernorm.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.16.self_attn.q_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.16.self_attn.k_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.16.self_attn.v_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.16.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.16.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.16.block_sparse_moe.experts.0.w3.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.16.block_sparse_moe.experts.1.w3.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.16.block_sparse_moe.experts.0.w1.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.16.block_sparse_moe.experts.1.w1.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.16.block_sparse_moe.experts.0.w2.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.16.block_sparse_moe.experts.1.w2.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.17.input_layernorm.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.17.self_attn.q_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.17.self_attn.k_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.17.self_attn.v_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.17.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.17.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.17.block_sparse_moe.experts.0.w3.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.17.block_sparse_moe.experts.1.w3.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.17.block_sparse_moe.experts.0.w1.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.17.block_sparse_moe.experts.1.w1.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.17.block_sparse_moe.experts.0.w2.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.17.block_sparse_moe.experts.1.w2.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.18.input_layernorm.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.18.self_attn.q_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.18.self_attn.k_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.18.self_attn.v_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.18.self_attn.o_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.18.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.18.block_sparse_moe.experts.0.w3.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.18.block_sparse_moe.experts.1.w3.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.18.block_sparse_moe.experts.0.w1.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.18.block_sparse_moe.experts.1.w1.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.18.block_sparse_moe.experts.0.w2.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.18.block_sparse_moe.experts.1.w2.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.19.input_layernorm.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.19.self_attn.q_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.19.self_attn.k_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.19.self_attn.v_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.19.self_attn.o_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.19.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.19.block_sparse_moe.experts.0.w3.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.19.block_sparse_moe.experts.1.w3.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.19.block_sparse_moe.experts.0.w1.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.19.block_sparse_moe.experts.1.w1.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.19.block_sparse_moe.experts.0.w2.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.19.block_sparse_moe.experts.1.w2.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.20.input_layernorm.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.20.self_attn.q_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.20.self_attn.k_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.20.self_attn.v_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.20.self_attn.o_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.20.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.20.block_sparse_moe.experts.0.w3.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.20.block_sparse_moe.experts.1.w3.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.20.block_sparse_moe.experts.0.w1.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.20.block_sparse_moe.experts.1.w1.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.20.block_sparse_moe.experts.0.w2.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.20.block_sparse_moe.experts.1.w2.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.21.input_layernorm.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.21.self_attn.q_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.21.self_attn.k_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.21.self_attn.v_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.21.self_attn.o_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.21.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.21.block_sparse_moe.experts.0.w3.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.21.block_sparse_moe.experts.1.w3.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.21.block_sparse_moe.experts.0.w1.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.21.block_sparse_moe.experts.1.w1.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.21.block_sparse_moe.experts.0.w2.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.21.block_sparse_moe.experts.1.w2.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.22.input_layernorm.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.22.self_attn.q_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.22.self_attn.k_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.22.self_attn.v_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.22.self_attn.o_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.22.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.22.block_sparse_moe.experts.0.w3.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.22.block_sparse_moe.experts.1.w3.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.22.block_sparse_moe.experts.0.w1.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.22.block_sparse_moe.experts.1.w1.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.22.block_sparse_moe.experts.0.w2.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.22.block_sparse_moe.experts.1.w2.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.23.input_layernorm.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.23.self_attn.q_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.23.self_attn.k_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.23.self_attn.v_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.23.self_attn.o_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.23.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.23.block_sparse_moe.experts.0.w3.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.23.block_sparse_moe.experts.1.w3.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.23.block_sparse_moe.experts.0.w1.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.23.block_sparse_moe.experts.1.w1.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.23.block_sparse_moe.experts.0.w2.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.23.block_sparse_moe.experts.1.w2.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.24.input_layernorm.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.24.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.24.self_attn.k_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.24.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.24.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.24.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.24.block_sparse_moe.experts.0.w3.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.24.block_sparse_moe.experts.1.w3.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.24.block_sparse_moe.experts.0.w1.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.24.block_sparse_moe.experts.1.w1.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.24.block_sparse_moe.experts.0.w2.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.24.block_sparse_moe.experts.1.w2.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.25.input_layernorm.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.25.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.25.self_attn.k_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.25.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.25.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.25.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.25.block_sparse_moe.experts.0.w3.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.25.block_sparse_moe.experts.1.w3.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.25.block_sparse_moe.experts.0.w1.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.25.block_sparse_moe.experts.1.w1.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.25.block_sparse_moe.experts.0.w2.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.25.block_sparse_moe.experts.1.w2.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.26.input_layernorm.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.26.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.26.self_attn.k_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.26.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.26.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.26.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.26.block_sparse_moe.experts.0.w3.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.26.block_sparse_moe.experts.1.w3.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.26.block_sparse_moe.experts.0.w1.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.26.block_sparse_moe.experts.1.w1.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.26.block_sparse_moe.experts.0.w2.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.26.block_sparse_moe.experts.1.w2.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.27.input_layernorm.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.27.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.27.self_attn.k_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.27.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.27.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.27.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.27.block_sparse_moe.experts.0.w3.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.27.block_sparse_moe.experts.1.w3.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.27.block_sparse_moe.experts.0.w1.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.27.block_sparse_moe.experts.1.w1.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.27.block_sparse_moe.experts.0.w2.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.27.block_sparse_moe.experts.1.w2.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.28.input_layernorm.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.28.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.28.self_attn.k_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.28.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.28.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.28.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.28.block_sparse_moe.experts.0.w3.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.28.block_sparse_moe.experts.1.w3.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.28.block_sparse_moe.experts.0.w1.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.28.block_sparse_moe.experts.1.w1.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.28.block_sparse_moe.experts.0.w2.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.28.block_sparse_moe.experts.1.w2.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.29.input_layernorm.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.29.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.29.self_attn.k_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.29.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.29.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.29.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.29.block_sparse_moe.experts.0.w3.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.29.block_sparse_moe.experts.1.w3.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.29.block_sparse_moe.experts.0.w1.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.29.block_sparse_moe.experts.1.w1.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.29.block_sparse_moe.experts.0.w2.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.29.block_sparse_moe.experts.1.w2.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.30.input_layernorm.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.30.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.30.self_attn.k_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.30.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.30.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.30.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.30.block_sparse_moe.experts.0.w3.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.30.block_sparse_moe.experts.1.w3.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.30.block_sparse_moe.experts.0.w1.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.30.block_sparse_moe.experts.1.w1.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.30.block_sparse_moe.experts.0.w2.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.30.block_sparse_moe.experts.1.w2.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.31.input_layernorm.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.31.self_attn.q_proj.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.31.self_attn.k_proj.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.31.self_attn.v_proj.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.31.self_attn.o_proj.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.31.post_attention_layernorm.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.31.block_sparse_moe.experts.0.w3.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.31.block_sparse_moe.experts.1.w3.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.31.block_sparse_moe.experts.0.w1.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.31.block_sparse_moe.experts.1.w1.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.31.block_sparse_moe.experts.0.w2.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.31.block_sparse_moe.experts.1.w2.weight": "model-00006-of-00006.safetensors",
|
||||
"model.norm.weight": "model-00006-of-00006.safetensors",
|
||||
"lm_head.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.0.block_sparse_moe.gate.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.1.block_sparse_moe.gate.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.2.block_sparse_moe.gate.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.3.block_sparse_moe.gate.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.4.block_sparse_moe.gate.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.5.block_sparse_moe.gate.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.6.block_sparse_moe.gate.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.7.block_sparse_moe.gate.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.8.block_sparse_moe.gate.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.9.block_sparse_moe.gate.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.10.block_sparse_moe.gate.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.11.block_sparse_moe.gate.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.12.block_sparse_moe.gate.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.13.block_sparse_moe.gate.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.14.block_sparse_moe.gate.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.15.block_sparse_moe.gate.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.16.block_sparse_moe.gate.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.17.block_sparse_moe.gate.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.18.block_sparse_moe.gate.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.19.block_sparse_moe.gate.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.20.block_sparse_moe.gate.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.21.block_sparse_moe.gate.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.22.block_sparse_moe.gate.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.23.block_sparse_moe.gate.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.24.block_sparse_moe.gate.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.25.block_sparse_moe.gate.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.26.block_sparse_moe.gate.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.27.block_sparse_moe.gate.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.28.block_sparse_moe.gate.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.29.block_sparse_moe.gate.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.30.block_sparse_moe.gate.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.31.block_sparse_moe.gate.weight": "model-00006-of-00006.safetensors"
|
||||
}
|
||||
}
|
||||
197
moe_defs.py
Normal file
197
moe_defs.py
Normal file
@@ -0,0 +1,197 @@
|
||||
# Copyright (C) 2025 Arcee AI
|
||||
# SPDX-License-Identifier: LGPL-3.0-only
|
||||
|
||||
from typing import ClassVar, List, Optional
|
||||
|
||||
from pydantic import BaseModel
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
from mergekit.architecture.base import (
|
||||
ModuleArchitecture,
|
||||
WeightInfo,
|
||||
)
|
||||
from mergekit.architecture.json_definitions import NAME_TO_ARCH
|
||||
|
||||
MISTRAL_INFO = NAME_TO_ARCH["MistralForCausalLM"][0]
|
||||
MISTRAL_MODULE_ARCH = MISTRAL_INFO.modules["default"].architecture
|
||||
|
||||
|
||||
class MixtralModuleArchitecture(ModuleArchitecture, BaseModel):
|
||||
ARCHITECTURE_NAME: ClassVar[str] = "MixtralForCausalLM"
|
||||
num_local_experts: int
|
||||
|
||||
def name(self) -> str:
|
||||
return "mixtral"
|
||||
|
||||
@classmethod
|
||||
def from_config(cls, config: PretrainedConfig):
|
||||
return MixtralModuleArchitecture(num_local_experts=config.num_local_experts)
|
||||
|
||||
def pre_weights(self, config: PretrainedConfig) -> List[WeightInfo]:
|
||||
return MISTRAL_MODULE_ARCH.pre_weights(config)
|
||||
|
||||
def post_weights(self, config: PretrainedConfig) -> List[WeightInfo]:
|
||||
return MISTRAL_MODULE_ARCH.post_weights(config)
|
||||
|
||||
def num_layers_config_key(self) -> str:
|
||||
return MISTRAL_MODULE_ARCH.num_layers_config_key()
|
||||
|
||||
def layer_weights(
|
||||
self, index: int, config: PretrainedConfig
|
||||
) -> Optional[List[WeightInfo]]:
|
||||
num_experts = self.num_local_experts
|
||||
prefix = f"model.layers.{index}"
|
||||
tensor_names = []
|
||||
for expert_idx in range(num_experts):
|
||||
for param in ("w1", "w2", "w3"):
|
||||
tensor_names.append(
|
||||
prefix + f".block_sparse_moe.experts.{expert_idx}.{param}.weight"
|
||||
)
|
||||
tensor_names.append(prefix + ".block_sparse_moe.gate.weight")
|
||||
res = []
|
||||
for name in tensor_names:
|
||||
res.append(WeightInfo(name=name))
|
||||
for weight_info in MISTRAL_MODULE_ARCH.layer_weights(index, config):
|
||||
if ".mlp." in weight_info.name:
|
||||
continue
|
||||
res.append(weight_info)
|
||||
return res
|
||||
|
||||
|
||||
QWEN3_INFO = NAME_TO_ARCH["Qwen3ForCausalLM"][0]
|
||||
QWEN3_MODULE_ARCH = QWEN3_INFO.modules["default"].architecture
|
||||
|
||||
|
||||
class Qwen3MoeModuleArchitecture(ModuleArchitecture, BaseModel):
|
||||
ARCHITECTURE_NAME: ClassVar[str] = "Qwen3MoeForCausalLM"
|
||||
num_experts: int
|
||||
|
||||
def name(self) -> str:
|
||||
return "qwen3_moe"
|
||||
|
||||
@classmethod
|
||||
def from_config(cls, config: PretrainedConfig):
|
||||
return Qwen3MoeModuleArchitecture(num_experts=config.num_experts)
|
||||
|
||||
def pre_weights(self, config: PretrainedConfig) -> List[WeightInfo]:
|
||||
return QWEN3_MODULE_ARCH.pre_weights(config)
|
||||
|
||||
def post_weights(self, config: PretrainedConfig) -> List[WeightInfo]:
|
||||
return QWEN3_MODULE_ARCH.post_weights(config)
|
||||
|
||||
def num_layers_config_key(self) -> str:
|
||||
return QWEN3_MODULE_ARCH.num_layers_config_key()
|
||||
|
||||
def layer_weights(
|
||||
self, index: int, config: PretrainedConfig
|
||||
) -> Optional[List[WeightInfo]]:
|
||||
prefix = f"model.layers.{index}"
|
||||
tensor_names = []
|
||||
for expert_idx in range(self.num_experts):
|
||||
for param in ("up_proj", "gate_proj", "down_proj"):
|
||||
tensor_names.append(
|
||||
prefix + f".mlp.experts.{expert_idx}.{param}.weight"
|
||||
)
|
||||
tensor_names.append(prefix + ".mlp.gate.weight")
|
||||
res = []
|
||||
for name in tensor_names:
|
||||
res.append(WeightInfo(name=name))
|
||||
for weight_info in QWEN3_MODULE_ARCH.layer_weights(index, config):
|
||||
if ".mlp." in weight_info.name:
|
||||
continue
|
||||
res.append(weight_info)
|
||||
return res
|
||||
|
||||
|
||||
AFMOE_PARTIAL_INFO = NAME_TO_ARCH["_AfmoePartialForCausalLM"][0]
|
||||
AFMOE_PARTIAL_MODULE_ARCH = AFMOE_PARTIAL_INFO.modules["default"].architecture
|
||||
|
||||
|
||||
class AfmoeModuleArchitecture(ModuleArchitecture, BaseModel):
|
||||
ARCHITECTURE_NAME: ClassVar[str] = "AfmoeForCausalLM"
|
||||
num_experts: int
|
||||
|
||||
def name(self) -> str:
|
||||
return "afmoe"
|
||||
|
||||
@classmethod
|
||||
def from_config(cls, config: PretrainedConfig):
|
||||
return AfmoeModuleArchitecture(num_experts=config.num_experts)
|
||||
|
||||
def pre_weights(self, config: PretrainedConfig) -> List[WeightInfo]:
|
||||
return AFMOE_PARTIAL_MODULE_ARCH.pre_weights(config)
|
||||
|
||||
def post_weights(self, config: PretrainedConfig) -> List[WeightInfo]:
|
||||
return AFMOE_PARTIAL_MODULE_ARCH.post_weights(config)
|
||||
|
||||
def num_layers_config_key(self) -> str:
|
||||
return AFMOE_PARTIAL_MODULE_ARCH.num_layers_config_key()
|
||||
|
||||
def layer_weights(
|
||||
self, index: int, config: PretrainedConfig
|
||||
) -> Optional[List[WeightInfo]]:
|
||||
res = AFMOE_PARTIAL_MODULE_ARCH.layer_weights(index, config) or []
|
||||
prefix = f"model.layers.{index}"
|
||||
for expert_idx in range(self.num_experts):
|
||||
for param in ("up_proj", "gate_proj", "down_proj"):
|
||||
res.append(
|
||||
WeightInfo(
|
||||
name=prefix + f".mlp.experts.{expert_idx}.{param}.weight",
|
||||
optional=True,
|
||||
)
|
||||
)
|
||||
return res
|
||||
|
||||
|
||||
# Add this to moe_defs.py
|
||||
|
||||
# 1. Get the base Llama info from the registry
|
||||
LLAMA_INFO = NAME_TO_ARCH["LlamaForCausalLM"][0]
|
||||
LLAMA_MODULE_ARCH = LLAMA_INFO.modules["default"].architecture
|
||||
|
||||
class LlamaMoeModuleArchitecture(ModuleArchitecture, BaseModel):
|
||||
# This is the name that will appear in the output config.json
|
||||
ARCHITECTURE_NAME: ClassVar[str] = "LlamaMoeForCausalLM"
|
||||
num_experts: int
|
||||
|
||||
def name(self) -> str:
|
||||
return "llama_moe"
|
||||
|
||||
@classmethod
|
||||
def from_config(cls, config: PretrainedConfig):
|
||||
# This looks for the 'num_experts' key in the model's config
|
||||
return LlamaMoeModuleArchitecture(num_experts=getattr(config, "num_experts", 8))
|
||||
|
||||
def pre_weights(self, config: PretrainedConfig) -> List[WeightInfo]:
|
||||
# Uses standard Llama embeddings/norms
|
||||
return LLAMA_MODULE_ARCH.pre_weights(config)
|
||||
|
||||
def post_weights(self, config: PretrainedConfig) -> List[WeightInfo]:
|
||||
# Uses standard Llama final norm/head
|
||||
return LLAMA_MODULE_ARCH.post_weights(config)
|
||||
|
||||
def num_layers_config_key(self) -> str:
|
||||
return LLAMA_MODULE_ARCH.num_layers_config_key()
|
||||
|
||||
def layer_weights(self, index: int, config: PretrainedConfig) -> Optional[List[WeightInfo]]:
|
||||
prefix = f"model.layers.{index}"
|
||||
res = []
|
||||
|
||||
# 2. Define the Expert weights
|
||||
# We map the dense MLP layers into an expert array
|
||||
for expert_idx in range(self.num_experts):
|
||||
for param in ("gate_proj", "up_proj", "down_proj"):
|
||||
res.append(
|
||||
WeightInfo(name=prefix + f".block_sparse_moe.experts.{expert_idx}.{param}.weight")
|
||||
)
|
||||
|
||||
# 3. Define the Router (Gate) weight
|
||||
res.append(WeightInfo(name=prefix + ".block_sparse_moe.gate.weight"))
|
||||
|
||||
# 4. Add the non-MLP weights (Attention layers, Input Norms)
|
||||
# We skip the original .mlp. weights because we replaced them with experts
|
||||
for weight_info in LLAMA_MODULE_ARCH.layer_weights(index, config):
|
||||
if ".mlp." not in weight_info.name:
|
||||
res.append(weight_info)
|
||||
|
||||
return res
|
||||
17
special_tokens_map.json
Normal file
17
special_tokens_map.json
Normal file
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"bos_token": {
|
||||
"content": "<|begin_of_text|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
},
|
||||
"eos_token": {
|
||||
"content": "<|eot_id|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
},
|
||||
"pad_token": "<|begin_of_text|>"
|
||||
}
|
||||
3
tokenizer.json
Normal file
3
tokenizer.json
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:71c1736d713a012aa4720d05a6c32c6992e9d278ea88cedd36008f14fd616116
|
||||
size 17210091
|
||||
2067
tokenizer_config.json
Normal file
2067
tokenizer_config.json
Normal file
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user