commit 726a7417a276c1a490f360ee645e4e4fc3f35a62
Author: ModelHub XC <noreply@modelhub.org.cn>
Date:   Mon Jul 6 08:10:17 2026 +0800

    初始化项目，由ModelHub XC社区提供模型
    
    Model: mkd-hossain/keural-dpo-final
    Source: Original Platform

diff --git a/.gitattributes b/.gitattributes
new file mode 100644
index 0000000..a6344aa
--- /dev/null
+++ b/.gitattributes
@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..c7c093e
--- /dev/null
+++ b/README.md
@@ -0,0 +1,284 @@
+---
+language:
+  - ko
+  - en
+license: apache-2.0
+library_name: transformers
+tags:
+  - mixtral
+  - moe
+  - korean
+  - bilingual
+  - causal-lm
+  - dpo
+  - rlhf
+  - instruction-tuned
+  - chat
+base_model: mkd-hossain/keural-sft-18k
+pipeline_tag: text-generation
+---
+
+# Keural-DPO-14.83B (Final — 6927 steps, 1 full epoch)
+
+Keural is a bilingual Korean–English Mixture-of-Experts language model trained **entirely from scratch** — no base model was used.
+This is the **final DPO (Direct Preference Optimization) checkpoint** at step 6,927, completing 1 full epoch of preference alignment from the Keural SFT-18k base.
+
+> This is the most capable Keural checkpoint released to date. One full epoch of DPO alignment on 440K Korean+English preference pairs, producing consistently positive reward margins throughout training.
+
+## Model Details
+
+| Property | Value |
+|---|---|
+| Architecture | Mixtral-style MoE (8 experts, top-2 routing) |
+| Parameters | **14.83B total** / ~7.42B active per token |
+| Layers | 24 |
+| Hidden size | 4096 |
+| Attention heads | 32 (GQA — 8 KV heads) |
+| Head dim | 128 |
+| Expert intermediate size | 5,632 |
+| Experts | 8 total, top-2 per token |
+| Context length | 4,096 tokens |
+| Vocabulary | 131,074 (131,072 SPM + `<|im_start|>` + `<|im_end|>`) |
+| RoPE theta | 500,000 |
+| Sliding window | 512 (alternating every other layer) |
+| Norm | RMSNorm (eps=1e-5) |
+| Activation | SiLU |
+| Dtype | bfloat16 |
+| Languages | Korean (primary), English |
+
+## Full Training Pipeline
+
+| Stage | Steps | Tokens | Data | Hardware |
+|---|---|---|---|---|
+| Pretraining Stage 1 | 100,000 | ~50B | Korean + English web corpus | 2× H200 SXM |
+| Pretraining Stage 2 | 120,000 | ~13B | Korean + English web corpus (continued) | 2× H200 SXM |
+| SFT | 18,000 | 710M | mkd-chanwoo/keural-SFT (1.14M ChatML samples) | 2× H200 SXM |
+| **DPO (this checkpoint)** | **6,927 (1 full epoch)** | — | keural-dpo-raw (440K preference pairs) | 2× H200 SXM |
+
+### DPO Training Details
+
+| Hyperparameter | Value |
+|---|---|
+| Algorithm | Direct Preference Optimization (DPO) |
+| Learning rate | 2e-6 → 2e-7 cosine decay |
+| Min learning rate | 2e-7 |
+| Warmup steps | 100 |
+| Beta (KL penalty) | 0.1 |
+| Batch size per GPU | 2 |
+| Gradient accumulation | 16 steps |
+| Effective batch size | 64 (2 × 16 × 2 GPUs) |
+| Max sequence length | 1,024 tokens |
+| Optimizer | AdamW (β1=0.9, β2=0.95, ε=1e-8) |
+| Weight decay | 0.1 |
+| Gradient clipping | 1.0 |
+| Total steps | 6,927 (1 full epoch) |
+| Dataset size | 440,627 preference pairs |
+| Parallelism | FSDP FULL_SHARD (ZeRO-3 equivalent) |
+| Precision | bfloat16 + gradient checkpointing |
+| Hardware | 2× NVIDIA H200 SXM (139 GiB each) |
+| Speed | ~40 seconds/step |
+| Final loss | ~0.6924 (stable) |
+| Final margin | +0.0009–0.0018 (consistently positive) |
+| Final GradNorm | 0.20–0.31 (clean) |
+
+### DPO Dataset Sources
+
+| Source | Samples | Language |
+|---|---|---|
+| hh_rlhf | 159,777 | English |
+| aihub_71760 | 116,320 | Korean |
+| multifaceted_collection_dpo | 63,399 | English |
+| ultrafeedback_binarized | 59,122 | English |
+| aihub_71748 | 29,676 | Korean |
+| orca_dpo_paris_ko | 12,714 | Korean |
+| **Total** | **440,627** | |
+
+### SFT Hyperparameters (base checkpoint)
+
+| Hyperparameter | Value |
+|---|---|
+| Learning rate | 1e-5 → 1e-6 cosine decay |
+| Effective batch size | 64 (4 per GPU × 8 grad accum × 2 GPUs) |
+| Max sequence length | 4,096 tokens |
+| Weight decay | 0.05 |
+| Steps | 18,000 |
+| Dataset | mkd-chanwoo/keural-SFT (1.14M samples) |
+
+## Chat Format (ChatML)
+
+This model uses **ChatML** format. Always include a system prompt for best results.
+
+```
+<|im_start|>system
+You are a helpful bilingual Korean-English assistant. Always respond in the same language as the user.<|im_end|>
+<|im_start|>user
+안녕하세요! 오늘 날씨가 어때요?<|im_end|>
+<|im_start|>assistant
+```
+
+The model generates until it produces `<|im_end|>` (token ID 131073).
+
+> The chat template in `tokenizer_config.json` **automatically injects a default system prompt** if you don't provide one, so bilingual behavior works out of the box with `apply_chat_template`.
+
+## How to Use
+
+### With `transformers`
+
+```python
+from transformers import AutoTokenizer, AutoModelForCausalLM
+import torch
+
+model_id = "mkd-hossain/keural-dpo-final"
+
+tokenizer = AutoTokenizer.from_pretrained(model_id)
+model = AutoModelForCausalLM.from_pretrained(
+    model_id,
+    torch_dtype=torch.bfloat16,
+    device_map="auto",
+)
+
+messages = [
+    {
+        "role": "system",
+        "content": (
+            "You are a helpful bilingual Korean-English assistant. "
+            "Always respond in the same language as the user's message."
+        )
+    },
+    {"role": "user", "content": "파이썬에서 리스트를 정렬하는 방법을 알려주세요."},
+]
+
+text = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
+inputs = tokenizer(text, return_tensors="pt").to(model.device)
+
+with torch.no_grad():
+    output = model.generate(
+        **inputs,
+        max_new_tokens=512,
+        temperature=0.7,
+        top_p=0.9,
+        top_k=50,
+        repetition_penalty=1.1,
+        no_repeat_ngram_size=8,
+        do_sample=True,
+        eos_token_id=131073,
+    )
+
+response = tokenizer.decode(output[0][inputs.input_ids.shape[1]:], skip_special_tokens=False)
+response = response.split("<|im_end|>")[0].strip()
+print(response)
+```
+
+### With vLLM (recommended for serving)
+
+```bash
+pip install vllm
+
+python -m vllm.entrypoints.openai.api_server \
+    --model mkd-hossain/keural-dpo-final \
+    --tokenizer mkd-hossain/keural-dpo-final \
+    --dtype bfloat16 \
+    --max-model-len 4096 \
+    --tensor-parallel-size 1
+```
+
+Call the OpenAI-compatible endpoint:
+
+```python
+from openai import OpenAI
+
+client = OpenAI(base_url="http://localhost:8000/v1", api_key="none")
+
+response = client.chat.completions.create(
+    model="mkd-hossain/keural-dpo-final",
+    messages=[
+        {"role": "system", "content": "You are a helpful bilingual assistant. Respond in the same language as the user."},
+        {"role": "user", "content": "What is the capital of South Korea?"},
+    ],
+    max_tokens=512,
+    temperature=0.7,
+)
+print(response.choices[0].message.content)
+```
+
+### Multi-GPU serving
+
+```bash
+python -m vllm.entrypoints.openai.api_server \
+    --model mkd-hossain/keural-dpo-final \
+    --dtype bfloat16 \
+    --max-model-len 4096 \
+    --tensor-parallel-size 2
+```
+
+### Manual ChatML prompt
+
+```python
+prompt = (
+    "<|im_start|>system\n"
+    "You are a helpful bilingual Korean-English assistant. "
+    "Always respond in the same language as the user.\n"
+    "<|im_end|>\n"
+    "<|im_start|>user\n"
+    "Tell me about Seoul.<|im_end|>\n"
+    "<|im_start|>assistant\n"
+)
+```
+
+## Special Tokens
+
+| Token | ID | Purpose |
+|---|---|---|
+| `<|im_start|>` | 131072 | Marks the start of each conversation turn |
+| `<|im_end|>` | 131073 | Marks end of turn / generation stop token |
+| `<bos>` | 1 | Beginning of sequence |
+| `<eos>` | 2 | End of sequence (not used for chat) |
+| `<pad>` | 0 | Padding token |
+
+> **Critical:** Always set `eos_token_id=131073` when generating. Do **not** use `eos_token_id=2`.
+
+## Recommended Generation Settings
+
+```python
+# Conversational / creative
+{
+    "max_new_tokens": 512,
+    "temperature": 0.7,
+    "top_p": 0.9,
+    "top_k": 50,
+    "repetition_penalty": 1.1,
+    "no_repeat_ngram_size": 8,
+    "do_sample": True,
+    "eos_token_id": 131073,
+}
+
+# Factual / deterministic
+{
+    "max_new_tokens": 512,
+    "temperature": 0.1,
+    "repetition_penalty": 1.1,
+    "do_sample": False,
+    "eos_token_id": 131073,
+}
+```
+
+## Checkpoint Comparison
+
+| Checkpoint | Stage | Steps | Notes |
+|---|---|---|---|
+| mkd-hossain/keural-pretrained | Pretraining | 120,000 | Raw base, no instruction tuning |
+| mkd-hossain/keural-sft-18k | SFT | 18,000 | Instruction following, ChatML format |
+| mkd-hossain/keural-dpo-3500 | DPO 50% | 3,500 | Early alignment |
+| mkd-hossain/keural-dpo-5500 | DPO 79% | 5,500 | Late alignment |
+| **mkd-hossain/keural-dpo-final** | **DPO 100%** | **6,927** | **Full epoch — best checkpoint** |
+
+## Limitations
+
+- Maximum context is 4,096 tokens.
+- The pretraining corpus is Korean-dominant — always include a system prompt for correct bilingual behavior.
+- Not safety-aligned — do not deploy in production without additional safety fine-tuning.
+- This is an intermediate model in an ongoing training pipeline. Future releases will include SFT epoch 2 on filtered data and DPO round 2.
+
+## License
+
+Apache 2.0
diff --git a/chat_template.jinja b/chat_template.jinja
new file mode 100644
index 0000000..00d0f15
--- /dev/null
+++ b/chat_template.jinja
@@ -0,0 +1,13 @@
+{% if messages[0]['role'] == 'system' %}
+{% set system_message = messages[0]['content'] %}
+{% set messages = messages[1:] %}
+{% else %}
+{% set system_message = 'You are a helpful bilingual Korean-English assistant. Always respond in the same language as the user.' %}
+{% endif %}
+<|im_start|>system
+{{ system_message }}<|im_end|>
+{% for message in messages %}
+<|im_start|>{{ message['role'] }}
+{{ message['content'] }}<|im_end|>
+{% endfor %}
+<|im_start|>assistant
diff --git a/config.json b/config.json
new file mode 100644
index 0000000..9fb8bba
--- /dev/null
+++ b/config.json
@@ -0,0 +1,32 @@
+{
+  "architectures": [
+    "KeuralMoECausalLM"
+  ],
+  "model_type": "keural",
+  "vocab_size": 131074,
+  "hidden_size": 4096,
+  "intermediate_size": 5632,
+  "num_hidden_layers": 24,
+  "num_attention_heads": 32,
+  "num_key_value_heads": 8,
+  "head_dim": 128,
+  "num_local_experts": 8,
+  "num_experts_per_tok": 2,
+  "max_position_embeddings": 4096,
+  "rope_theta": 500000.0,
+  "rms_norm_eps": 1e-05,
+  "sliding_window": 512,
+  "hidden_act": "silu",
+  "initializer_range": 0.02,
+  "use_cache": true,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.40.0",
+  "output_router_logits": false,
+  "router_aux_loss_coef": 0.001,
+  "keural_step": 6927,
+  "auto_map": {
+    "AutoConfig": "configuration_keural.KeuralConfig",
+    "AutoModelForCausalLM": "modeling_keural.KeuralMoECausalLM"
+  }
+}
\ No newline at end of file
diff --git a/configuration_keural.py b/configuration_keural.py
new file mode 100644
index 0000000..11a9bbd
--- /dev/null
+++ b/configuration_keural.py
@@ -0,0 +1,13 @@
+"""
+KeuralConfig — custom configuration class for Keural MoE.
+Registered as model_type "keural" so HuggingFace AutoConfig can resolve it.
+"""
+
+from transformers import MixtralConfig
+
+
+class KeuralConfig(MixtralConfig):
+    model_type = "keural"
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
diff --git a/generation_config.json b/generation_config.json
new file mode 100644
index 0000000..7f1a8c3
--- /dev/null
+++ b/generation_config.json
@@ -0,0 +1,12 @@
+{
+  "bos_token_id": 1,
+  "eos_token_id": 131073,
+  "pad_token_id": 0,
+  "max_new_tokens": 512,
+  "temperature": 0.7,
+  "top_p": 0.9,
+  "top_k": 50,
+  "repetition_penalty": 1.1,
+  "do_sample": true,
+  "transformers_version": "4.40.0"
+}
diff --git a/model-00001-of-00007.safetensors b/model-00001-of-00007.safetensors
new file mode 100644
index 0000000..3c6b792
--- /dev/null
+++ b/model-00001-of-00007.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4f3e2bb1cca5ac31cf2cae3a8e22abf89dc222003dd153113287d97c90061154
+size 4823806664
diff --git a/model-00002-of-00007.safetensors b/model-00002-of-00007.safetensors
new file mode 100644
index 0000000..0c9bd2f
--- /dev/null
+++ b/model-00002-of-00007.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:269cdcf92b5b2823d717101c55c31ed8e4cea7465c431348820624e80ce7f5a9
+size 4811210208
diff --git a/model-00003-of-00007.safetensors b/model-00003-of-00007.safetensors
new file mode 100644
index 0000000..52db85f
--- /dev/null
+++ b/model-00003-of-00007.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ba19f2f3bed2fa70f02c4363541a07af39c64c8215f30eb5c863011cfb89c00c
+size 4811210248
diff --git a/model-00004-of-00007.safetensors b/model-00004-of-00007.safetensors
new file mode 100644
index 0000000..3946d6c
--- /dev/null
+++ b/model-00004-of-00007.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:295054e8c2501b02c7af79808deef1256e574e98d647e790843bb5346aa51890
+size 4811210336
diff --git a/model-00005-of-00007.safetensors b/model-00005-of-00007.safetensors
new file mode 100644
index 0000000..8984d97
--- /dev/null
+++ b/model-00005-of-00007.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:41bafd9ab61deb1d145622ee33b7f94d935dc0a349168ceb75a4c67f198d1f70
+size 4811210336
diff --git a/model-00006-of-00007.safetensors b/model-00006-of-00007.safetensors
new file mode 100644
index 0000000..c47960a
--- /dev/null
+++ b/model-00006-of-00007.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cc0e1ed04564bc1db2617cc707071245bb83312e020c708af5e20dc8856a84d0
+size 4811210336
diff --git a/model-00007-of-00007.safetensors b/model-00007-of-00007.safetensors
new file mode 100644
index 0000000..7e4d608
--- /dev/null
+++ b/model-00007-of-00007.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fcabcda05c1f80101512ea369eb4d09823a62577f3f707a33bd2411d8f3c6a56
+size 1858103664
diff --git a/model.safetensors.index.json b/model.safetensors.index.json
new file mode 100644
index 0000000..f6ee7fb
--- /dev/null
+++ b/model.safetensors.index.json
@@ -0,0 +1,754 @@
+{
+  "metadata": {
+    "total_size": 30737866752
+  },
+  "weight_map": {
+    "model.embed_tokens.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.block_sparse_moe.gate.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.block_sparse_moe.experts.0.w1.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.block_sparse_moe.experts.0.w3.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.block_sparse_moe.experts.0.w2.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.block_sparse_moe.experts.1.w1.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.block_sparse_moe.experts.1.w3.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.block_sparse_moe.experts.1.w2.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.block_sparse_moe.experts.2.w1.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.block_sparse_moe.experts.2.w3.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.block_sparse_moe.experts.2.w2.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.block_sparse_moe.experts.3.w1.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.block_sparse_moe.experts.3.w3.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.block_sparse_moe.experts.3.w2.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.block_sparse_moe.experts.4.w1.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.block_sparse_moe.experts.4.w3.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.block_sparse_moe.experts.4.w2.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.block_sparse_moe.experts.5.w1.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.block_sparse_moe.experts.5.w3.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.block_sparse_moe.experts.5.w2.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.block_sparse_moe.experts.6.w1.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.block_sparse_moe.experts.6.w3.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.block_sparse_moe.experts.6.w2.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.block_sparse_moe.experts.7.w1.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.block_sparse_moe.experts.7.w3.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.block_sparse_moe.experts.7.w2.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.block_sparse_moe.gate.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.block_sparse_moe.experts.0.w1.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.block_sparse_moe.experts.0.w3.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.block_sparse_moe.experts.0.w2.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.block_sparse_moe.experts.1.w1.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.block_sparse_moe.experts.1.w3.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.block_sparse_moe.experts.1.w2.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.block_sparse_moe.experts.2.w1.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.block_sparse_moe.experts.2.w3.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.block_sparse_moe.experts.2.w2.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.block_sparse_moe.experts.3.w1.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.block_sparse_moe.experts.3.w3.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.block_sparse_moe.experts.3.w2.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.block_sparse_moe.experts.4.w1.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.block_sparse_moe.experts.4.w3.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.block_sparse_moe.experts.4.w2.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.block_sparse_moe.experts.5.w1.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.block_sparse_moe.experts.5.w3.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.block_sparse_moe.experts.5.w2.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.block_sparse_moe.experts.6.w1.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.block_sparse_moe.experts.6.w3.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.block_sparse_moe.experts.6.w2.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.block_sparse_moe.experts.7.w1.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.block_sparse_moe.experts.7.w3.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.block_sparse_moe.experts.7.w2.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.block_sparse_moe.gate.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.block_sparse_moe.experts.0.w1.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.block_sparse_moe.experts.0.w3.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.block_sparse_moe.experts.0.w2.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.block_sparse_moe.experts.1.w1.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.block_sparse_moe.experts.1.w3.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.block_sparse_moe.experts.1.w2.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.block_sparse_moe.experts.2.w1.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.block_sparse_moe.experts.2.w3.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.block_sparse_moe.experts.2.w2.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.block_sparse_moe.experts.3.w1.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.block_sparse_moe.experts.3.w3.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.block_sparse_moe.experts.3.w2.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.block_sparse_moe.experts.4.w1.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.block_sparse_moe.experts.4.w3.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.block_sparse_moe.experts.4.w2.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.block_sparse_moe.experts.5.w1.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.block_sparse_moe.experts.5.w3.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.block_sparse_moe.experts.5.w2.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.block_sparse_moe.experts.6.w1.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.block_sparse_moe.experts.6.w3.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.block_sparse_moe.experts.6.w2.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.block_sparse_moe.experts.7.w1.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.block_sparse_moe.experts.7.w3.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.block_sparse_moe.experts.7.w2.weight": "model-00001-of-00007.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.3.block_sparse_moe.gate.weight": "model-00001-of-00007.safetensors",
+    "model.layers.3.block_sparse_moe.experts.0.w1.weight": "model-00001-of-00007.safetensors",
+    "model.layers.3.block_sparse_moe.experts.0.w3.weight": "model-00001-of-00007.safetensors",
+    "model.layers.3.block_sparse_moe.experts.0.w2.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.block_sparse_moe.experts.1.w1.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.block_sparse_moe.experts.1.w3.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.block_sparse_moe.experts.1.w2.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.block_sparse_moe.experts.2.w1.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.block_sparse_moe.experts.2.w3.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.block_sparse_moe.experts.2.w2.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.block_sparse_moe.experts.3.w1.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.block_sparse_moe.experts.3.w3.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.block_sparse_moe.experts.3.w2.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.block_sparse_moe.experts.4.w1.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.block_sparse_moe.experts.4.w3.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.block_sparse_moe.experts.4.w2.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.block_sparse_moe.experts.5.w1.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.block_sparse_moe.experts.5.w3.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.block_sparse_moe.experts.5.w2.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.block_sparse_moe.experts.6.w1.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.block_sparse_moe.experts.6.w3.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.block_sparse_moe.experts.6.w2.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.block_sparse_moe.experts.7.w1.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.block_sparse_moe.experts.7.w3.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.block_sparse_moe.experts.7.w2.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.block_sparse_moe.gate.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.block_sparse_moe.experts.0.w1.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.block_sparse_moe.experts.0.w3.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.block_sparse_moe.experts.0.w2.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.block_sparse_moe.experts.1.w1.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.block_sparse_moe.experts.1.w3.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.block_sparse_moe.experts.1.w2.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.block_sparse_moe.experts.2.w1.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.block_sparse_moe.experts.2.w3.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.block_sparse_moe.experts.2.w2.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.block_sparse_moe.experts.3.w1.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.block_sparse_moe.experts.3.w3.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.block_sparse_moe.experts.3.w2.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.block_sparse_moe.experts.4.w1.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.block_sparse_moe.experts.4.w3.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.block_sparse_moe.experts.4.w2.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.block_sparse_moe.experts.5.w1.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.block_sparse_moe.experts.5.w3.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.block_sparse_moe.experts.5.w2.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.block_sparse_moe.experts.6.w1.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.block_sparse_moe.experts.6.w3.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.block_sparse_moe.experts.6.w2.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.block_sparse_moe.experts.7.w1.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.block_sparse_moe.experts.7.w3.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.block_sparse_moe.experts.7.w2.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.block_sparse_moe.gate.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.block_sparse_moe.experts.0.w1.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.block_sparse_moe.experts.0.w3.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.block_sparse_moe.experts.0.w2.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.block_sparse_moe.experts.1.w1.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.block_sparse_moe.experts.1.w3.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.block_sparse_moe.experts.1.w2.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.block_sparse_moe.experts.2.w1.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.block_sparse_moe.experts.2.w3.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.block_sparse_moe.experts.2.w2.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.block_sparse_moe.experts.3.w1.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.block_sparse_moe.experts.3.w3.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.block_sparse_moe.experts.3.w2.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.block_sparse_moe.experts.4.w1.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.block_sparse_moe.experts.4.w3.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.block_sparse_moe.experts.4.w2.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.block_sparse_moe.experts.5.w1.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.block_sparse_moe.experts.5.w3.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.block_sparse_moe.experts.5.w2.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.block_sparse_moe.experts.6.w1.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.block_sparse_moe.experts.6.w3.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.block_sparse_moe.experts.6.w2.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.block_sparse_moe.experts.7.w1.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.block_sparse_moe.experts.7.w3.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.block_sparse_moe.experts.7.w2.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.block_sparse_moe.gate.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.block_sparse_moe.experts.0.w1.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.block_sparse_moe.experts.0.w3.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.block_sparse_moe.experts.0.w2.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.block_sparse_moe.experts.1.w1.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.block_sparse_moe.experts.1.w3.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.block_sparse_moe.experts.1.w2.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.block_sparse_moe.experts.2.w1.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.block_sparse_moe.experts.2.w3.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.block_sparse_moe.experts.2.w2.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.block_sparse_moe.experts.3.w1.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.block_sparse_moe.experts.3.w3.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.block_sparse_moe.experts.3.w2.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.block_sparse_moe.experts.4.w1.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.block_sparse_moe.experts.4.w3.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.block_sparse_moe.experts.4.w2.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.block_sparse_moe.experts.5.w1.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.block_sparse_moe.experts.5.w3.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.block_sparse_moe.experts.5.w2.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.block_sparse_moe.experts.6.w1.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.block_sparse_moe.experts.6.w3.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.block_sparse_moe.experts.6.w2.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.block_sparse_moe.experts.7.w1.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.block_sparse_moe.experts.7.w3.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.block_sparse_moe.experts.7.w2.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.block_sparse_moe.gate.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.block_sparse_moe.experts.0.w1.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.block_sparse_moe.experts.0.w3.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.block_sparse_moe.experts.0.w2.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.block_sparse_moe.experts.1.w1.weight": "model-00003-of-00007.safetensors",
+    "model.layers.7.block_sparse_moe.experts.1.w3.weight": "model-00003-of-00007.safetensors",
+    "model.layers.7.block_sparse_moe.experts.1.w2.weight": "model-00003-of-00007.safetensors",
+    "model.layers.7.block_sparse_moe.experts.2.w1.weight": "model-00003-of-00007.safetensors",
+    "model.layers.7.block_sparse_moe.experts.2.w3.weight": "model-00003-of-00007.safetensors",
+    "model.layers.7.block_sparse_moe.experts.2.w2.weight": "model-00003-of-00007.safetensors",
+    "model.layers.7.block_sparse_moe.experts.3.w1.weight": "model-00003-of-00007.safetensors",
+    "model.layers.7.block_sparse_moe.experts.3.w3.weight": "model-00003-of-00007.safetensors",
+    "model.layers.7.block_sparse_moe.experts.3.w2.weight": "model-00003-of-00007.safetensors",
+    "model.layers.7.block_sparse_moe.experts.4.w1.weight": "model-00003-of-00007.safetensors",
+    "model.layers.7.block_sparse_moe.experts.4.w3.weight": "model-00003-of-00007.safetensors",
+    "model.layers.7.block_sparse_moe.experts.4.w2.weight": "model-00003-of-00007.safetensors",
+    "model.layers.7.block_sparse_moe.experts.5.w1.weight": "model-00003-of-00007.safetensors",
+    "model.layers.7.block_sparse_moe.experts.5.w3.weight": "model-00003-of-00007.safetensors",
+    "model.layers.7.block_sparse_moe.experts.5.w2.weight": "model-00003-of-00007.safetensors",
+    "model.layers.7.block_sparse_moe.experts.6.w1.weight": "model-00003-of-00007.safetensors",
+    "model.layers.7.block_sparse_moe.experts.6.w3.weight": "model-00003-of-00007.safetensors",
+    "model.layers.7.block_sparse_moe.experts.6.w2.weight": "model-00003-of-00007.safetensors",
+    "model.layers.7.block_sparse_moe.experts.7.w1.weight": "model-00003-of-00007.safetensors",
+    "model.layers.7.block_sparse_moe.experts.7.w3.weight": "model-00003-of-00007.safetensors",
+    "model.layers.7.block_sparse_moe.experts.7.w2.weight": "model-00003-of-00007.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.8.block_sparse_moe.gate.weight": "model-00003-of-00007.safetensors",
+    "model.layers.8.block_sparse_moe.experts.0.w1.weight": "model-00003-of-00007.safetensors",
+    "model.layers.8.block_sparse_moe.experts.0.w3.weight": "model-00003-of-00007.safetensors",
+    "model.layers.8.block_sparse_moe.experts.0.w2.weight": "model-00003-of-00007.safetensors",
+    "model.layers.8.block_sparse_moe.experts.1.w1.weight": "model-00003-of-00007.safetensors",
+    "model.layers.8.block_sparse_moe.experts.1.w3.weight": "model-00003-of-00007.safetensors",
+    "model.layers.8.block_sparse_moe.experts.1.w2.weight": "model-00003-of-00007.safetensors",
+    "model.layers.8.block_sparse_moe.experts.2.w1.weight": "model-00003-of-00007.safetensors",
+    "model.layers.8.block_sparse_moe.experts.2.w3.weight": "model-00003-of-00007.safetensors",
+    "model.layers.8.block_sparse_moe.experts.2.w2.weight": "model-00003-of-00007.safetensors",
+    "model.layers.8.block_sparse_moe.experts.3.w1.weight": "model-00003-of-00007.safetensors",
+    "model.layers.8.block_sparse_moe.experts.3.w3.weight": "model-00003-of-00007.safetensors",
+    "model.layers.8.block_sparse_moe.experts.3.w2.weight": "model-00003-of-00007.safetensors",
+    "model.layers.8.block_sparse_moe.experts.4.w1.weight": "model-00003-of-00007.safetensors",
+    "model.layers.8.block_sparse_moe.experts.4.w3.weight": "model-00003-of-00007.safetensors",
+    "model.layers.8.block_sparse_moe.experts.4.w2.weight": "model-00003-of-00007.safetensors",
+    "model.layers.8.block_sparse_moe.experts.5.w1.weight": "model-00003-of-00007.safetensors",
+    "model.layers.8.block_sparse_moe.experts.5.w3.weight": "model-00003-of-00007.safetensors",
+    "model.layers.8.block_sparse_moe.experts.5.w2.weight": "model-00003-of-00007.safetensors",
+    "model.layers.8.block_sparse_moe.experts.6.w1.weight": "model-00003-of-00007.safetensors",
+    "model.layers.8.block_sparse_moe.experts.6.w3.weight": "model-00003-of-00007.safetensors",
+    "model.layers.8.block_sparse_moe.experts.6.w2.weight": "model-00003-of-00007.safetensors",
+    "model.layers.8.block_sparse_moe.experts.7.w1.weight": "model-00003-of-00007.safetensors",
+    "model.layers.8.block_sparse_moe.experts.7.w3.weight": "model-00003-of-00007.safetensors",
+    "model.layers.8.block_sparse_moe.experts.7.w2.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.block_sparse_moe.gate.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.block_sparse_moe.experts.0.w1.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.block_sparse_moe.experts.0.w3.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.block_sparse_moe.experts.0.w2.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.block_sparse_moe.experts.1.w1.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.block_sparse_moe.experts.1.w3.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.block_sparse_moe.experts.1.w2.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.block_sparse_moe.experts.2.w1.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.block_sparse_moe.experts.2.w3.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.block_sparse_moe.experts.2.w2.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.block_sparse_moe.experts.3.w1.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.block_sparse_moe.experts.3.w3.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.block_sparse_moe.experts.3.w2.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.block_sparse_moe.experts.4.w1.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.block_sparse_moe.experts.4.w3.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.block_sparse_moe.experts.4.w2.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.block_sparse_moe.experts.5.w1.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.block_sparse_moe.experts.5.w3.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.block_sparse_moe.experts.5.w2.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.block_sparse_moe.experts.6.w1.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.block_sparse_moe.experts.6.w3.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.block_sparse_moe.experts.6.w2.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.block_sparse_moe.experts.7.w1.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.block_sparse_moe.experts.7.w3.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.block_sparse_moe.experts.7.w2.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.block_sparse_moe.gate.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.block_sparse_moe.experts.0.w1.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.block_sparse_moe.experts.0.w3.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.block_sparse_moe.experts.0.w2.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.block_sparse_moe.experts.1.w1.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.block_sparse_moe.experts.1.w3.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.block_sparse_moe.experts.1.w2.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.block_sparse_moe.experts.2.w1.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.block_sparse_moe.experts.2.w3.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.block_sparse_moe.experts.2.w2.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.block_sparse_moe.experts.3.w1.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.block_sparse_moe.experts.3.w3.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.block_sparse_moe.experts.3.w2.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.block_sparse_moe.experts.4.w1.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.block_sparse_moe.experts.4.w3.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.block_sparse_moe.experts.4.w2.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.block_sparse_moe.experts.5.w1.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.block_sparse_moe.experts.5.w3.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.block_sparse_moe.experts.5.w2.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.block_sparse_moe.experts.6.w1.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.block_sparse_moe.experts.6.w3.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.block_sparse_moe.experts.6.w2.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.block_sparse_moe.experts.7.w1.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.block_sparse_moe.experts.7.w3.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.block_sparse_moe.experts.7.w2.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.block_sparse_moe.gate.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.block_sparse_moe.experts.0.w1.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.block_sparse_moe.experts.0.w3.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.block_sparse_moe.experts.0.w2.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.block_sparse_moe.experts.1.w1.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.block_sparse_moe.experts.1.w3.weight": "model-00004-of-00007.safetensors",
+    "model.layers.11.block_sparse_moe.experts.1.w2.weight": "model-00004-of-00007.safetensors",
+    "model.layers.11.block_sparse_moe.experts.2.w1.weight": "model-00004-of-00007.safetensors",
+    "model.layers.11.block_sparse_moe.experts.2.w3.weight": "model-00004-of-00007.safetensors",
+    "model.layers.11.block_sparse_moe.experts.2.w2.weight": "model-00004-of-00007.safetensors",
+    "model.layers.11.block_sparse_moe.experts.3.w1.weight": "model-00004-of-00007.safetensors",
+    "model.layers.11.block_sparse_moe.experts.3.w3.weight": "model-00004-of-00007.safetensors",
+    "model.layers.11.block_sparse_moe.experts.3.w2.weight": "model-00004-of-00007.safetensors",
+    "model.layers.11.block_sparse_moe.experts.4.w1.weight": "model-00004-of-00007.safetensors",
+    "model.layers.11.block_sparse_moe.experts.4.w3.weight": "model-00004-of-00007.safetensors",
+    "model.layers.11.block_sparse_moe.experts.4.w2.weight": "model-00004-of-00007.safetensors",
+    "model.layers.11.block_sparse_moe.experts.5.w1.weight": "model-00004-of-00007.safetensors",
+    "model.layers.11.block_sparse_moe.experts.5.w3.weight": "model-00004-of-00007.safetensors",
+    "model.layers.11.block_sparse_moe.experts.5.w2.weight": "model-00004-of-00007.safetensors",
+    "model.layers.11.block_sparse_moe.experts.6.w1.weight": "model-00004-of-00007.safetensors",
+    "model.layers.11.block_sparse_moe.experts.6.w3.weight": "model-00004-of-00007.safetensors",
+    "model.layers.11.block_sparse_moe.experts.6.w2.weight": "model-00004-of-00007.safetensors",
+    "model.layers.11.block_sparse_moe.experts.7.w1.weight": "model-00004-of-00007.safetensors",
+    "model.layers.11.block_sparse_moe.experts.7.w3.weight": "model-00004-of-00007.safetensors",
+    "model.layers.11.block_sparse_moe.experts.7.w2.weight": "model-00004-of-00007.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.12.block_sparse_moe.gate.weight": "model-00004-of-00007.safetensors",
+    "model.layers.12.block_sparse_moe.experts.0.w1.weight": "model-00004-of-00007.safetensors",
+    "model.layers.12.block_sparse_moe.experts.0.w3.weight": "model-00004-of-00007.safetensors",
+    "model.layers.12.block_sparse_moe.experts.0.w2.weight": "model-00004-of-00007.safetensors",
+    "model.layers.12.block_sparse_moe.experts.1.w1.weight": "model-00004-of-00007.safetensors",
+    "model.layers.12.block_sparse_moe.experts.1.w3.weight": "model-00004-of-00007.safetensors",
+    "model.layers.12.block_sparse_moe.experts.1.w2.weight": "model-00004-of-00007.safetensors",
+    "model.layers.12.block_sparse_moe.experts.2.w1.weight": "model-00004-of-00007.safetensors",
+    "model.layers.12.block_sparse_moe.experts.2.w3.weight": "model-00004-of-00007.safetensors",
+    "model.layers.12.block_sparse_moe.experts.2.w2.weight": "model-00004-of-00007.safetensors",
+    "model.layers.12.block_sparse_moe.experts.3.w1.weight": "model-00004-of-00007.safetensors",
+    "model.layers.12.block_sparse_moe.experts.3.w3.weight": "model-00004-of-00007.safetensors",
+    "model.layers.12.block_sparse_moe.experts.3.w2.weight": "model-00004-of-00007.safetensors",
+    "model.layers.12.block_sparse_moe.experts.4.w1.weight": "model-00004-of-00007.safetensors",
+    "model.layers.12.block_sparse_moe.experts.4.w3.weight": "model-00004-of-00007.safetensors",
+    "model.layers.12.block_sparse_moe.experts.4.w2.weight": "model-00004-of-00007.safetensors",
+    "model.layers.12.block_sparse_moe.experts.5.w1.weight": "model-00004-of-00007.safetensors",
+    "model.layers.12.block_sparse_moe.experts.5.w3.weight": "model-00004-of-00007.safetensors",
+    "model.layers.12.block_sparse_moe.experts.5.w2.weight": "model-00004-of-00007.safetensors",
+    "model.layers.12.block_sparse_moe.experts.6.w1.weight": "model-00004-of-00007.safetensors",
+    "model.layers.12.block_sparse_moe.experts.6.w3.weight": "model-00004-of-00007.safetensors",
+    "model.layers.12.block_sparse_moe.experts.6.w2.weight": "model-00004-of-00007.safetensors",
+    "model.layers.12.block_sparse_moe.experts.7.w1.weight": "model-00004-of-00007.safetensors",
+    "model.layers.12.block_sparse_moe.experts.7.w3.weight": "model-00004-of-00007.safetensors",
+    "model.layers.12.block_sparse_moe.experts.7.w2.weight": "model-00004-of-00007.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.13.block_sparse_moe.gate.weight": "model-00004-of-00007.safetensors",
+    "model.layers.13.block_sparse_moe.experts.0.w1.weight": "model-00004-of-00007.safetensors",
+    "model.layers.13.block_sparse_moe.experts.0.w3.weight": "model-00004-of-00007.safetensors",
+    "model.layers.13.block_sparse_moe.experts.0.w2.weight": "model-00004-of-00007.safetensors",
+    "model.layers.13.block_sparse_moe.experts.1.w1.weight": "model-00004-of-00007.safetensors",
+    "model.layers.13.block_sparse_moe.experts.1.w3.weight": "model-00004-of-00007.safetensors",
+    "model.layers.13.block_sparse_moe.experts.1.w2.weight": "model-00004-of-00007.safetensors",
+    "model.layers.13.block_sparse_moe.experts.2.w1.weight": "model-00004-of-00007.safetensors",
+    "model.layers.13.block_sparse_moe.experts.2.w3.weight": "model-00004-of-00007.safetensors",
+    "model.layers.13.block_sparse_moe.experts.2.w2.weight": "model-00004-of-00007.safetensors",
+    "model.layers.13.block_sparse_moe.experts.3.w1.weight": "model-00004-of-00007.safetensors",
+    "model.layers.13.block_sparse_moe.experts.3.w3.weight": "model-00004-of-00007.safetensors",
+    "model.layers.13.block_sparse_moe.experts.3.w2.weight": "model-00004-of-00007.safetensors",
+    "model.layers.13.block_sparse_moe.experts.4.w1.weight": "model-00004-of-00007.safetensors",
+    "model.layers.13.block_sparse_moe.experts.4.w3.weight": "model-00004-of-00007.safetensors",
+    "model.layers.13.block_sparse_moe.experts.4.w2.weight": "model-00004-of-00007.safetensors",
+    "model.layers.13.block_sparse_moe.experts.5.w1.weight": "model-00004-of-00007.safetensors",
+    "model.layers.13.block_sparse_moe.experts.5.w3.weight": "model-00004-of-00007.safetensors",
+    "model.layers.13.block_sparse_moe.experts.5.w2.weight": "model-00004-of-00007.safetensors",
+    "model.layers.13.block_sparse_moe.experts.6.w1.weight": "model-00004-of-00007.safetensors",
+    "model.layers.13.block_sparse_moe.experts.6.w3.weight": "model-00004-of-00007.safetensors",
+    "model.layers.13.block_sparse_moe.experts.6.w2.weight": "model-00004-of-00007.safetensors",
+    "model.layers.13.block_sparse_moe.experts.7.w1.weight": "model-00004-of-00007.safetensors",
+    "model.layers.13.block_sparse_moe.experts.7.w3.weight": "model-00004-of-00007.safetensors",
+    "model.layers.13.block_sparse_moe.experts.7.w2.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.block_sparse_moe.gate.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.block_sparse_moe.experts.0.w1.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.block_sparse_moe.experts.0.w3.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.block_sparse_moe.experts.0.w2.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.block_sparse_moe.experts.1.w1.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.block_sparse_moe.experts.1.w3.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.block_sparse_moe.experts.1.w2.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.block_sparse_moe.experts.2.w1.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.block_sparse_moe.experts.2.w3.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.block_sparse_moe.experts.2.w2.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.block_sparse_moe.experts.3.w1.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.block_sparse_moe.experts.3.w3.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.block_sparse_moe.experts.3.w2.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.block_sparse_moe.experts.4.w1.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.block_sparse_moe.experts.4.w3.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.block_sparse_moe.experts.4.w2.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.block_sparse_moe.experts.5.w1.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.block_sparse_moe.experts.5.w3.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.block_sparse_moe.experts.5.w2.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.block_sparse_moe.experts.6.w1.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.block_sparse_moe.experts.6.w3.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.block_sparse_moe.experts.6.w2.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.block_sparse_moe.experts.7.w1.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.block_sparse_moe.experts.7.w3.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.block_sparse_moe.experts.7.w2.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.block_sparse_moe.gate.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.block_sparse_moe.experts.0.w1.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.block_sparse_moe.experts.0.w3.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.block_sparse_moe.experts.0.w2.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.block_sparse_moe.experts.1.w1.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.block_sparse_moe.experts.1.w3.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.block_sparse_moe.experts.1.w2.weight": "model-00005-of-00007.safetensors",
+    "model.layers.15.block_sparse_moe.experts.2.w1.weight": "model-00005-of-00007.safetensors",
+    "model.layers.15.block_sparse_moe.experts.2.w3.weight": "model-00005-of-00007.safetensors",
+    "model.layers.15.block_sparse_moe.experts.2.w2.weight": "model-00005-of-00007.safetensors",
+    "model.layers.15.block_sparse_moe.experts.3.w1.weight": "model-00005-of-00007.safetensors",
+    "model.layers.15.block_sparse_moe.experts.3.w3.weight": "model-00005-of-00007.safetensors",
+    "model.layers.15.block_sparse_moe.experts.3.w2.weight": "model-00005-of-00007.safetensors",
+    "model.layers.15.block_sparse_moe.experts.4.w1.weight": "model-00005-of-00007.safetensors",
+    "model.layers.15.block_sparse_moe.experts.4.w3.weight": "model-00005-of-00007.safetensors",
+    "model.layers.15.block_sparse_moe.experts.4.w2.weight": "model-00005-of-00007.safetensors",
+    "model.layers.15.block_sparse_moe.experts.5.w1.weight": "model-00005-of-00007.safetensors",
+    "model.layers.15.block_sparse_moe.experts.5.w3.weight": "model-00005-of-00007.safetensors",
+    "model.layers.15.block_sparse_moe.experts.5.w2.weight": "model-00005-of-00007.safetensors",
+    "model.layers.15.block_sparse_moe.experts.6.w1.weight": "model-00005-of-00007.safetensors",
+    "model.layers.15.block_sparse_moe.experts.6.w3.weight": "model-00005-of-00007.safetensors",
+    "model.layers.15.block_sparse_moe.experts.6.w2.weight": "model-00005-of-00007.safetensors",
+    "model.layers.15.block_sparse_moe.experts.7.w1.weight": "model-00005-of-00007.safetensors",
+    "model.layers.15.block_sparse_moe.experts.7.w3.weight": "model-00005-of-00007.safetensors",
+    "model.layers.15.block_sparse_moe.experts.7.w2.weight": "model-00005-of-00007.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.16.block_sparse_moe.gate.weight": "model-00005-of-00007.safetensors",
+    "model.layers.16.block_sparse_moe.experts.0.w1.weight": "model-00005-of-00007.safetensors",
+    "model.layers.16.block_sparse_moe.experts.0.w3.weight": "model-00005-of-00007.safetensors",
+    "model.layers.16.block_sparse_moe.experts.0.w2.weight": "model-00005-of-00007.safetensors",
+    "model.layers.16.block_sparse_moe.experts.1.w1.weight": "model-00005-of-00007.safetensors",
+    "model.layers.16.block_sparse_moe.experts.1.w3.weight": "model-00005-of-00007.safetensors",
+    "model.layers.16.block_sparse_moe.experts.1.w2.weight": "model-00005-of-00007.safetensors",
+    "model.layers.16.block_sparse_moe.experts.2.w1.weight": "model-00005-of-00007.safetensors",
+    "model.layers.16.block_sparse_moe.experts.2.w3.weight": "model-00005-of-00007.safetensors",
+    "model.layers.16.block_sparse_moe.experts.2.w2.weight": "model-00005-of-00007.safetensors",
+    "model.layers.16.block_sparse_moe.experts.3.w1.weight": "model-00005-of-00007.safetensors",
+    "model.layers.16.block_sparse_moe.experts.3.w3.weight": "model-00005-of-00007.safetensors",
+    "model.layers.16.block_sparse_moe.experts.3.w2.weight": "model-00005-of-00007.safetensors",
+    "model.layers.16.block_sparse_moe.experts.4.w1.weight": "model-00005-of-00007.safetensors",
+    "model.layers.16.block_sparse_moe.experts.4.w3.weight": "model-00005-of-00007.safetensors",
+    "model.layers.16.block_sparse_moe.experts.4.w2.weight": "model-00005-of-00007.safetensors",
+    "model.layers.16.block_sparse_moe.experts.5.w1.weight": "model-00005-of-00007.safetensors",
+    "model.layers.16.block_sparse_moe.experts.5.w3.weight": "model-00005-of-00007.safetensors",
+    "model.layers.16.block_sparse_moe.experts.5.w2.weight": "model-00005-of-00007.safetensors",
+    "model.layers.16.block_sparse_moe.experts.6.w1.weight": "model-00005-of-00007.safetensors",
+    "model.layers.16.block_sparse_moe.experts.6.w3.weight": "model-00005-of-00007.safetensors",
+    "model.layers.16.block_sparse_moe.experts.6.w2.weight": "model-00005-of-00007.safetensors",
+    "model.layers.16.block_sparse_moe.experts.7.w1.weight": "model-00005-of-00007.safetensors",
+    "model.layers.16.block_sparse_moe.experts.7.w3.weight": "model-00005-of-00007.safetensors",
+    "model.layers.16.block_sparse_moe.experts.7.w2.weight": "model-00005-of-00007.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.17.block_sparse_moe.gate.weight": "model-00005-of-00007.safetensors",
+    "model.layers.17.block_sparse_moe.experts.0.w1.weight": "model-00005-of-00007.safetensors",
+    "model.layers.17.block_sparse_moe.experts.0.w3.weight": "model-00005-of-00007.safetensors",
+    "model.layers.17.block_sparse_moe.experts.0.w2.weight": "model-00005-of-00007.safetensors",
+    "model.layers.17.block_sparse_moe.experts.1.w1.weight": "model-00005-of-00007.safetensors",
+    "model.layers.17.block_sparse_moe.experts.1.w3.weight": "model-00005-of-00007.safetensors",
+    "model.layers.17.block_sparse_moe.experts.1.w2.weight": "model-00005-of-00007.safetensors",
+    "model.layers.17.block_sparse_moe.experts.2.w1.weight": "model-00005-of-00007.safetensors",
+    "model.layers.17.block_sparse_moe.experts.2.w3.weight": "model-00005-of-00007.safetensors",
+    "model.layers.17.block_sparse_moe.experts.2.w2.weight": "model-00005-of-00007.safetensors",
+    "model.layers.17.block_sparse_moe.experts.3.w1.weight": "model-00005-of-00007.safetensors",
+    "model.layers.17.block_sparse_moe.experts.3.w3.weight": "model-00005-of-00007.safetensors",
+    "model.layers.17.block_sparse_moe.experts.3.w2.weight": "model-00005-of-00007.safetensors",
+    "model.layers.17.block_sparse_moe.experts.4.w1.weight": "model-00005-of-00007.safetensors",
+    "model.layers.17.block_sparse_moe.experts.4.w3.weight": "model-00005-of-00007.safetensors",
+    "model.layers.17.block_sparse_moe.experts.4.w2.weight": "model-00005-of-00007.safetensors",
+    "model.layers.17.block_sparse_moe.experts.5.w1.weight": "model-00005-of-00007.safetensors",
+    "model.layers.17.block_sparse_moe.experts.5.w3.weight": "model-00005-of-00007.safetensors",
+    "model.layers.17.block_sparse_moe.experts.5.w2.weight": "model-00005-of-00007.safetensors",
+    "model.layers.17.block_sparse_moe.experts.6.w1.weight": "model-00005-of-00007.safetensors",
+    "model.layers.17.block_sparse_moe.experts.6.w3.weight": "model-00005-of-00007.safetensors",
+    "model.layers.17.block_sparse_moe.experts.6.w2.weight": "model-00005-of-00007.safetensors",
+    "model.layers.17.block_sparse_moe.experts.7.w1.weight": "model-00005-of-00007.safetensors",
+    "model.layers.17.block_sparse_moe.experts.7.w3.weight": "model-00005-of-00007.safetensors",
+    "model.layers.17.block_sparse_moe.experts.7.w2.weight": "model-00005-of-00007.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.18.block_sparse_moe.gate.weight": "model-00005-of-00007.safetensors",
+    "model.layers.18.block_sparse_moe.experts.0.w1.weight": "model-00005-of-00007.safetensors",
+    "model.layers.18.block_sparse_moe.experts.0.w3.weight": "model-00005-of-00007.safetensors",
+    "model.layers.18.block_sparse_moe.experts.0.w2.weight": "model-00005-of-00007.safetensors",
+    "model.layers.18.block_sparse_moe.experts.1.w1.weight": "model-00005-of-00007.safetensors",
+    "model.layers.18.block_sparse_moe.experts.1.w3.weight": "model-00005-of-00007.safetensors",
+    "model.layers.18.block_sparse_moe.experts.1.w2.weight": "model-00005-of-00007.safetensors",
+    "model.layers.18.block_sparse_moe.experts.2.w1.weight": "model-00005-of-00007.safetensors",
+    "model.layers.18.block_sparse_moe.experts.2.w3.weight": "model-00005-of-00007.safetensors",
+    "model.layers.18.block_sparse_moe.experts.2.w2.weight": "model-00005-of-00007.safetensors",
+    "model.layers.18.block_sparse_moe.experts.3.w1.weight": "model-00005-of-00007.safetensors",
+    "model.layers.18.block_sparse_moe.experts.3.w3.weight": "model-00005-of-00007.safetensors",
+    "model.layers.18.block_sparse_moe.experts.3.w2.weight": "model-00005-of-00007.safetensors",
+    "model.layers.18.block_sparse_moe.experts.4.w1.weight": "model-00005-of-00007.safetensors",
+    "model.layers.18.block_sparse_moe.experts.4.w3.weight": "model-00005-of-00007.safetensors",
+    "model.layers.18.block_sparse_moe.experts.4.w2.weight": "model-00005-of-00007.safetensors",
+    "model.layers.18.block_sparse_moe.experts.5.w1.weight": "model-00005-of-00007.safetensors",
+    "model.layers.18.block_sparse_moe.experts.5.w3.weight": "model-00005-of-00007.safetensors",
+    "model.layers.18.block_sparse_moe.experts.5.w2.weight": "model-00005-of-00007.safetensors",
+    "model.layers.18.block_sparse_moe.experts.6.w1.weight": "model-00005-of-00007.safetensors",
+    "model.layers.18.block_sparse_moe.experts.6.w3.weight": "model-00005-of-00007.safetensors",
+    "model.layers.18.block_sparse_moe.experts.6.w2.weight": "model-00005-of-00007.safetensors",
+    "model.layers.18.block_sparse_moe.experts.7.w1.weight": "model-00005-of-00007.safetensors",
+    "model.layers.18.block_sparse_moe.experts.7.w3.weight": "model-00005-of-00007.safetensors",
+    "model.layers.18.block_sparse_moe.experts.7.w2.weight": "model-00005-of-00007.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.19.block_sparse_moe.gate.weight": "model-00005-of-00007.safetensors",
+    "model.layers.19.block_sparse_moe.experts.0.w1.weight": "model-00005-of-00007.safetensors",
+    "model.layers.19.block_sparse_moe.experts.0.w3.weight": "model-00005-of-00007.safetensors",
+    "model.layers.19.block_sparse_moe.experts.0.w2.weight": "model-00005-of-00007.safetensors",
+    "model.layers.19.block_sparse_moe.experts.1.w1.weight": "model-00005-of-00007.safetensors",
+    "model.layers.19.block_sparse_moe.experts.1.w3.weight": "model-00005-of-00007.safetensors",
+    "model.layers.19.block_sparse_moe.experts.1.w2.weight": "model-00005-of-00007.safetensors",
+    "model.layers.19.block_sparse_moe.experts.2.w1.weight": "model-00006-of-00007.safetensors",
+    "model.layers.19.block_sparse_moe.experts.2.w3.weight": "model-00006-of-00007.safetensors",
+    "model.layers.19.block_sparse_moe.experts.2.w2.weight": "model-00006-of-00007.safetensors",
+    "model.layers.19.block_sparse_moe.experts.3.w1.weight": "model-00006-of-00007.safetensors",
+    "model.layers.19.block_sparse_moe.experts.3.w3.weight": "model-00006-of-00007.safetensors",
+    "model.layers.19.block_sparse_moe.experts.3.w2.weight": "model-00006-of-00007.safetensors",
+    "model.layers.19.block_sparse_moe.experts.4.w1.weight": "model-00006-of-00007.safetensors",
+    "model.layers.19.block_sparse_moe.experts.4.w3.weight": "model-00006-of-00007.safetensors",
+    "model.layers.19.block_sparse_moe.experts.4.w2.weight": "model-00006-of-00007.safetensors",
+    "model.layers.19.block_sparse_moe.experts.5.w1.weight": "model-00006-of-00007.safetensors",
+    "model.layers.19.block_sparse_moe.experts.5.w3.weight": "model-00006-of-00007.safetensors",
+    "model.layers.19.block_sparse_moe.experts.5.w2.weight": "model-00006-of-00007.safetensors",
+    "model.layers.19.block_sparse_moe.experts.6.w1.weight": "model-00006-of-00007.safetensors",
+    "model.layers.19.block_sparse_moe.experts.6.w3.weight": "model-00006-of-00007.safetensors",
+    "model.layers.19.block_sparse_moe.experts.6.w2.weight": "model-00006-of-00007.safetensors",
+    "model.layers.19.block_sparse_moe.experts.7.w1.weight": "model-00006-of-00007.safetensors",
+    "model.layers.19.block_sparse_moe.experts.7.w3.weight": "model-00006-of-00007.safetensors",
+    "model.layers.19.block_sparse_moe.experts.7.w2.weight": "model-00006-of-00007.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.20.block_sparse_moe.gate.weight": "model-00006-of-00007.safetensors",
+    "model.layers.20.block_sparse_moe.experts.0.w1.weight": "model-00006-of-00007.safetensors",
+    "model.layers.20.block_sparse_moe.experts.0.w3.weight": "model-00006-of-00007.safetensors",
+    "model.layers.20.block_sparse_moe.experts.0.w2.weight": "model-00006-of-00007.safetensors",
+    "model.layers.20.block_sparse_moe.experts.1.w1.weight": "model-00006-of-00007.safetensors",
+    "model.layers.20.block_sparse_moe.experts.1.w3.weight": "model-00006-of-00007.safetensors",
+    "model.layers.20.block_sparse_moe.experts.1.w2.weight": "model-00006-of-00007.safetensors",
+    "model.layers.20.block_sparse_moe.experts.2.w1.weight": "model-00006-of-00007.safetensors",
+    "model.layers.20.block_sparse_moe.experts.2.w3.weight": "model-00006-of-00007.safetensors",
+    "model.layers.20.block_sparse_moe.experts.2.w2.weight": "model-00006-of-00007.safetensors",
+    "model.layers.20.block_sparse_moe.experts.3.w1.weight": "model-00006-of-00007.safetensors",
+    "model.layers.20.block_sparse_moe.experts.3.w3.weight": "model-00006-of-00007.safetensors",
+    "model.layers.20.block_sparse_moe.experts.3.w2.weight": "model-00006-of-00007.safetensors",
+    "model.layers.20.block_sparse_moe.experts.4.w1.weight": "model-00006-of-00007.safetensors",
+    "model.layers.20.block_sparse_moe.experts.4.w3.weight": "model-00006-of-00007.safetensors",
+    "model.layers.20.block_sparse_moe.experts.4.w2.weight": "model-00006-of-00007.safetensors",
+    "model.layers.20.block_sparse_moe.experts.5.w1.weight": "model-00006-of-00007.safetensors",
+    "model.layers.20.block_sparse_moe.experts.5.w3.weight": "model-00006-of-00007.safetensors",
+    "model.layers.20.block_sparse_moe.experts.5.w2.weight": "model-00006-of-00007.safetensors",
+    "model.layers.20.block_sparse_moe.experts.6.w1.weight": "model-00006-of-00007.safetensors",
+    "model.layers.20.block_sparse_moe.experts.6.w3.weight": "model-00006-of-00007.safetensors",
+    "model.layers.20.block_sparse_moe.experts.6.w2.weight": "model-00006-of-00007.safetensors",
+    "model.layers.20.block_sparse_moe.experts.7.w1.weight": "model-00006-of-00007.safetensors",
+    "model.layers.20.block_sparse_moe.experts.7.w3.weight": "model-00006-of-00007.safetensors",
+    "model.layers.20.block_sparse_moe.experts.7.w2.weight": "model-00006-of-00007.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.21.block_sparse_moe.gate.weight": "model-00006-of-00007.safetensors",
+    "model.layers.21.block_sparse_moe.experts.0.w1.weight": "model-00006-of-00007.safetensors",
+    "model.layers.21.block_sparse_moe.experts.0.w3.weight": "model-00006-of-00007.safetensors",
+    "model.layers.21.block_sparse_moe.experts.0.w2.weight": "model-00006-of-00007.safetensors",
+    "model.layers.21.block_sparse_moe.experts.1.w1.weight": "model-00006-of-00007.safetensors",
+    "model.layers.21.block_sparse_moe.experts.1.w3.weight": "model-00006-of-00007.safetensors",
+    "model.layers.21.block_sparse_moe.experts.1.w2.weight": "model-00006-of-00007.safetensors",
+    "model.layers.21.block_sparse_moe.experts.2.w1.weight": "model-00006-of-00007.safetensors",
+    "model.layers.21.block_sparse_moe.experts.2.w3.weight": "model-00006-of-00007.safetensors",
+    "model.layers.21.block_sparse_moe.experts.2.w2.weight": "model-00006-of-00007.safetensors",
+    "model.layers.21.block_sparse_moe.experts.3.w1.weight": "model-00006-of-00007.safetensors",
+    "model.layers.21.block_sparse_moe.experts.3.w3.weight": "model-00006-of-00007.safetensors",
+    "model.layers.21.block_sparse_moe.experts.3.w2.weight": "model-00006-of-00007.safetensors",
+    "model.layers.21.block_sparse_moe.experts.4.w1.weight": "model-00006-of-00007.safetensors",
+    "model.layers.21.block_sparse_moe.experts.4.w3.weight": "model-00006-of-00007.safetensors",
+    "model.layers.21.block_sparse_moe.experts.4.w2.weight": "model-00006-of-00007.safetensors",
+    "model.layers.21.block_sparse_moe.experts.5.w1.weight": "model-00006-of-00007.safetensors",
+    "model.layers.21.block_sparse_moe.experts.5.w3.weight": "model-00006-of-00007.safetensors",
+    "model.layers.21.block_sparse_moe.experts.5.w2.weight": "model-00006-of-00007.safetensors",
+    "model.layers.21.block_sparse_moe.experts.6.w1.weight": "model-00006-of-00007.safetensors",
+    "model.layers.21.block_sparse_moe.experts.6.w3.weight": "model-00006-of-00007.safetensors",
+    "model.layers.21.block_sparse_moe.experts.6.w2.weight": "model-00006-of-00007.safetensors",
+    "model.layers.21.block_sparse_moe.experts.7.w1.weight": "model-00006-of-00007.safetensors",
+    "model.layers.21.block_sparse_moe.experts.7.w3.weight": "model-00006-of-00007.safetensors",
+    "model.layers.21.block_sparse_moe.experts.7.w2.weight": "model-00006-of-00007.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.22.block_sparse_moe.gate.weight": "model-00006-of-00007.safetensors",
+    "model.layers.22.block_sparse_moe.experts.0.w1.weight": "model-00006-of-00007.safetensors",
+    "model.layers.22.block_sparse_moe.experts.0.w3.weight": "model-00006-of-00007.safetensors",
+    "model.layers.22.block_sparse_moe.experts.0.w2.weight": "model-00006-of-00007.safetensors",
+    "model.layers.22.block_sparse_moe.experts.1.w1.weight": "model-00006-of-00007.safetensors",
+    "model.layers.22.block_sparse_moe.experts.1.w3.weight": "model-00006-of-00007.safetensors",
+    "model.layers.22.block_sparse_moe.experts.1.w2.weight": "model-00006-of-00007.safetensors",
+    "model.layers.22.block_sparse_moe.experts.2.w1.weight": "model-00006-of-00007.safetensors",
+    "model.layers.22.block_sparse_moe.experts.2.w3.weight": "model-00006-of-00007.safetensors",
+    "model.layers.22.block_sparse_moe.experts.2.w2.weight": "model-00006-of-00007.safetensors",
+    "model.layers.22.block_sparse_moe.experts.3.w1.weight": "model-00006-of-00007.safetensors",
+    "model.layers.22.block_sparse_moe.experts.3.w3.weight": "model-00006-of-00007.safetensors",
+    "model.layers.22.block_sparse_moe.experts.3.w2.weight": "model-00006-of-00007.safetensors",
+    "model.layers.22.block_sparse_moe.experts.4.w1.weight": "model-00006-of-00007.safetensors",
+    "model.layers.22.block_sparse_moe.experts.4.w3.weight": "model-00006-of-00007.safetensors",
+    "model.layers.22.block_sparse_moe.experts.4.w2.weight": "model-00006-of-00007.safetensors",
+    "model.layers.22.block_sparse_moe.experts.5.w1.weight": "model-00006-of-00007.safetensors",
+    "model.layers.22.block_sparse_moe.experts.5.w3.weight": "model-00006-of-00007.safetensors",
+    "model.layers.22.block_sparse_moe.experts.5.w2.weight": "model-00006-of-00007.safetensors",
+    "model.layers.22.block_sparse_moe.experts.6.w1.weight": "model-00006-of-00007.safetensors",
+    "model.layers.22.block_sparse_moe.experts.6.w3.weight": "model-00006-of-00007.safetensors",
+    "model.layers.22.block_sparse_moe.experts.6.w2.weight": "model-00006-of-00007.safetensors",
+    "model.layers.22.block_sparse_moe.experts.7.w1.weight": "model-00006-of-00007.safetensors",
+    "model.layers.22.block_sparse_moe.experts.7.w3.weight": "model-00006-of-00007.safetensors",
+    "model.layers.22.block_sparse_moe.experts.7.w2.weight": "model-00006-of-00007.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.23.block_sparse_moe.gate.weight": "model-00006-of-00007.safetensors",
+    "model.layers.23.block_sparse_moe.experts.0.w1.weight": "model-00006-of-00007.safetensors",
+    "model.layers.23.block_sparse_moe.experts.0.w3.weight": "model-00006-of-00007.safetensors",
+    "model.layers.23.block_sparse_moe.experts.0.w2.weight": "model-00006-of-00007.safetensors",
+    "model.layers.23.block_sparse_moe.experts.1.w1.weight": "model-00006-of-00007.safetensors",
+    "model.layers.23.block_sparse_moe.experts.1.w3.weight": "model-00006-of-00007.safetensors",
+    "model.layers.23.block_sparse_moe.experts.1.w2.weight": "model-00006-of-00007.safetensors",
+    "model.layers.23.block_sparse_moe.experts.2.w1.weight": "model-00006-of-00007.safetensors",
+    "model.layers.23.block_sparse_moe.experts.2.w3.weight": "model-00007-of-00007.safetensors",
+    "model.layers.23.block_sparse_moe.experts.2.w2.weight": "model-00007-of-00007.safetensors",
+    "model.layers.23.block_sparse_moe.experts.3.w1.weight": "model-00007-of-00007.safetensors",
+    "model.layers.23.block_sparse_moe.experts.3.w3.weight": "model-00007-of-00007.safetensors",
+    "model.layers.23.block_sparse_moe.experts.3.w2.weight": "model-00007-of-00007.safetensors",
+    "model.layers.23.block_sparse_moe.experts.4.w1.weight": "model-00007-of-00007.safetensors",
+    "model.layers.23.block_sparse_moe.experts.4.w3.weight": "model-00007-of-00007.safetensors",
+    "model.layers.23.block_sparse_moe.experts.4.w2.weight": "model-00007-of-00007.safetensors",
+    "model.layers.23.block_sparse_moe.experts.5.w1.weight": "model-00007-of-00007.safetensors",
+    "model.layers.23.block_sparse_moe.experts.5.w3.weight": "model-00007-of-00007.safetensors",
+    "model.layers.23.block_sparse_moe.experts.5.w2.weight": "model-00007-of-00007.safetensors",
+    "model.layers.23.block_sparse_moe.experts.6.w1.weight": "model-00007-of-00007.safetensors",
+    "model.layers.23.block_sparse_moe.experts.6.w3.weight": "model-00007-of-00007.safetensors",
+    "model.layers.23.block_sparse_moe.experts.6.w2.weight": "model-00007-of-00007.safetensors",
+    "model.layers.23.block_sparse_moe.experts.7.w1.weight": "model-00007-of-00007.safetensors",
+    "model.layers.23.block_sparse_moe.experts.7.w3.weight": "model-00007-of-00007.safetensors",
+    "model.layers.23.block_sparse_moe.experts.7.w2.weight": "model-00007-of-00007.safetensors",
+    "model.norm.weight": "model-00007-of-00007.safetensors",
+    "lm_head.weight": "model-00007-of-00007.safetensors"
+  }
+}
\ No newline at end of file
diff --git a/modeling_keural.py b/modeling_keural.py
new file mode 100644
index 0000000..6857824
--- /dev/null
+++ b/modeling_keural.py
@@ -0,0 +1,43 @@
+"""
+KeuralMoECausalLM — Keural Mixture-of-Experts Causal Language Model.
+
+Bilingual Korean-English MoE LLM trained entirely from scratch.
+Developed by MKD Corp AI Research, Republic of Korea.
+
+Architecture:
+  - 14.83B total parameters (~7.42B active per token)
+  - 24 layers, hidden=4096, GQA 32/8 heads
+  - 8 experts per layer, top-2 routing
+  - Sliding window attention (512, alternating layers)
+  - RoPE theta=500,000, context length=4096
+  - Vocabulary: 131,074 tokens (131,072 SPM + <|im_start|> + <|im_end|>)
+
+Load with trust_remote_code=True:
+  from transformers import AutoModelForCausalLM, AutoTokenizer
+  model = AutoModelForCausalLM.from_pretrained(
+      "mkd-hossain/keural-sft3-final",
+      torch_dtype="bfloat16",
+      device_map="auto",
+      trust_remote_code=True,
+  )
+"""
+
+from transformers import MixtralForCausalLM
+from transformers.utils import logging
+
+try:
+    from configuration_keural import KeuralConfig
+except ImportError:
+    from .configuration_keural import KeuralConfig
+
+logger = logging.get_logger(__name__)
+
+
+class KeuralMoECausalLM(MixtralForCausalLM):
+    """
+    Keural MoE Causal Language Model.
+    Bilingual Korean-English 14.83B MoE LLM trained from scratch by MKD Corp AI Research.
+    """
+
+    config_class = KeuralConfig
+    _keys_to_ignore_on_load_missing = ["lm_head.weight"]
diff --git a/special_tokens_map.json b/special_tokens_map.json
new file mode 100644
index 0000000..3dee8b8
--- /dev/null
+++ b/special_tokens_map.json
@@ -0,0 +1,6 @@
+{
+  "bos_token": {"content": "<bos>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false},
+  "eos_token": {"content": "<|im_end|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false},
+  "unk_token": {"content": "<unk>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false},
+  "pad_token": {"content": "<pad>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false}
+}
diff --git a/tokenizer.model b/tokenizer.model
new file mode 100644
index 0000000..1b292cd
--- /dev/null
+++ b/tokenizer.model
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b982818ea2f2057ba791e2006d17683799f1d8ceb9c91322018a638c4ec4b170
+size 2657284
diff --git a/tokenizer_config.json b/tokenizer_config.json
new file mode 100644
index 0000000..382c000
--- /dev/null
+++ b/tokenizer_config.json
@@ -0,0 +1,16 @@
+{
+  "bos_token": "<bos>",
+  "eos_token": "<|im_end|>",
+  "unk_token": "<unk>",
+  "pad_token": "<pad>",
+  "model_max_length": 4096,
+  "tokenizer_class": "LlamaTokenizer",
+  "sp_model_kwargs": {},
+  "legacy": false,
+  "chat_template": "{% if messages[0]['role'] == 'system' %}{% set system_message = messages[0]['content'] %}{% set messages = messages[1:] %}{% else %}{% set system_message = 'You are a helpful bilingual Korean-English assistant. Always respond in the same language as the user.' %}{% endif %}<|im_start|>system\n{{ system_message }}<|im_end|>\n{% for message in messages %}<|im_start|>{{ message['role'] }}\n{{ message['content'] }}<|im_end|>\n{% endfor %}<|im_start|>assistant\n",
+  "added_tokens_decoder": {
+    "131072": {"content": "<|im_start|>", "special": true, "single_word": false, "lstrip": false, "rstrip": false, "normalized": false},
+    "131073": {"content": "<|im_end|>", "special": true, "single_word": false, "lstrip": false, "rstrip": false, "normalized": false}
+  },
+  "additional_special_tokens": ["<|im_start|>", "<|im_end|>"]
+}