From 03f2020a2be474e0686a18573b6ac32271592d70 Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Fri, 26 Jun 2026 08:06:16 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: mkd-hossain/keural-dpo-5500 Source: Original Platform --- .gitattributes | 35 ++ README.md | 289 ++++++++++++ config.json | 32 ++ configuration_keural.py | 13 + generation_config.json | 12 + model-00001-of-00007.safetensors | 3 + model-00002-of-00007.safetensors | 3 + model-00003-of-00007.safetensors | 3 + model-00004-of-00007.safetensors | 3 + model-00005-of-00007.safetensors | 3 + model-00006-of-00007.safetensors | 3 + model-00007-of-00007.safetensors | 3 + model.safetensors.index.json | 754 +++++++++++++++++++++++++++++++ modeling_keural.py | 43 ++ special_tokens_map.json | 30 ++ tokenizer.model | 3 + tokenizer_config.json | 16 + 17 files changed, 1248 insertions(+) create mode 100644 .gitattributes create mode 100644 README.md create mode 100644 config.json create mode 100644 configuration_keural.py create mode 100644 generation_config.json create mode 100644 model-00001-of-00007.safetensors create mode 100644 model-00002-of-00007.safetensors create mode 100644 model-00003-of-00007.safetensors create mode 100644 model-00004-of-00007.safetensors create mode 100644 model-00005-of-00007.safetensors create mode 100644 model-00006-of-00007.safetensors create mode 100644 model-00007-of-00007.safetensors create mode 100644 model.safetensors.index.json create mode 100644 modeling_keural.py create mode 100644 special_tokens_map.json create mode 100644 tokenizer.model create mode 100644 tokenizer_config.json diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..a6344aa --- /dev/null +++ b/.gitattributes @@ -0,0 +1,35 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..aa86ee0 --- /dev/null +++ b/README.md @@ -0,0 +1,289 @@ +--- +language: + - ko + - en +license: apache-2.0 +library_name: transformers +tags: + - mixtral + - moe + - korean + - bilingual + - causal-lm + - dpo + - rlhf + - instruction-tuned + - chat +base_model: mkd-hossain/keural-sft-18k +pipeline_tag: text-generation +--- + +# Keural-DPO-14.83B (checkpoint 5500) + +Keural is a bilingual Korean–English Mixture-of-Experts language model trained **entirely from scratch** — no base model was used. +This is the **DPO (Direct Preference Optimization) checkpoint** at step 5,500 (~79% of 1 epoch), aligned from the Keural SFT-18k base using human preference data. + +> This checkpoint is more mature than the 3500-step release. At step 5500 the model has seen ~80% of the full preference dataset, producing noticeably better instruction-following and more consistent language matching compared to the SFT base. + +## Model Details + +| Property | Value | +|---|---| +| Architecture | Mixtral-style MoE (8 experts, top-2 routing) | +| Parameters | **14.83B total** / ~7.42B active per token | +| Layers | 24 | +| Hidden size | 4096 | +| Attention heads | 32 (GQA — 8 KV heads) | +| KV heads | 8 | +| Head dim | 128 | +| Expert intermediate size | 5,632 | +| Experts | 8 total, top-2 per token | +| Context length | 4,096 tokens | +| Vocabulary | 131,074 (131,072 SPM + `<|im_start|>` + `<|im_end|>`) | +| RoPE theta | 500,000 | +| Sliding window | 512 (alternating every other layer) | +| Norm | RMSNorm (eps=1e-5) | +| Activation | SiLU | +| Dtype | bfloat16 | +| Languages | Korean (primary), English | + +## Full Training Pipeline + +| Stage | Steps | Tokens | Data | Hardware | +|---|---|---|---|---| +| Pretraining Stage 1 | 100,000 | ~50B | Korean + English web corpus | 2× H200 SXM | +| Pretraining Stage 2 | 120,000 | ~13B | Korean + English web corpus (continued) | 2× H200 SXM | +| SFT | 18,000 | 710M | mkd-chanwoo/keural-SFT (1.14M ChatML samples) | 2× H200 SXM | +| **DPO (this checkpoint)** | **5,500 / 6,927** | — | keural-dpo-raw (440K preference pairs) | 2× H200 SXM | + +### DPO Training Details + +| Hyperparameter | Value | +|---|---| +| Algorithm | Direct Preference Optimization (DPO) | +| Learning rate | 2e-6 → 2e-7 cosine decay | +| Min learning rate | 2e-7 | +| LR at step 5500 | ~3.87e-7 | +| Warmup steps | 100 | +| Beta (KL penalty) | 0.1 | +| Batch size per GPU | 2 | +| Gradient accumulation | 16 steps | +| Effective batch size | 64 (2 × 16 × 2 GPUs) | +| Max sequence length | 1,024 tokens | +| Optimizer | AdamW (β1=0.9, β2=0.95, ε=1e-8) | +| Weight decay | 0.1 | +| Gradient clipping | 1.0 | +| Total steps (1 epoch) | 6,927 | +| Dataset size | 440,627 preference pairs | +| Parallelism | FSDP FULL_SHARD (ZeRO-3 equivalent) | +| Precision | bfloat16 + gradient checkpointing | +| Hardware | 2× NVIDIA H200 SXM (139 GiB each) | +| Speed | ~40 seconds/step | + +**DPO loss at step 5500:** ~0.6924 (stable) +**Margin at step 5500:** +0.0009 to +0.0018 (consistently positive — model reliably prefers chosen responses) +**GradNorm:** 0.20–0.31 (clean, no explosion) + +### SFT Hyperparameters (base checkpoint) + +| Hyperparameter | Value | +|---|---| +| Learning rate | 1e-5 → 1e-6 cosine decay | +| Effective batch size | 64 (4 per GPU × 8 grad accum × 2 GPUs) | +| Max sequence length | 4,096 tokens | +| Weight decay | 0.05 | +| Steps | 18,000 | +| Dataset | mkd-chanwoo/keural-SFT (1.14M samples) | + +## Chat Format (ChatML) + +This model uses **ChatML** format. You **must** use this exact format for good results. + +``` +<|im_start|>system +You are a helpful bilingual Korean-English assistant. Always respond in the same language as the user.<|im_end|> +<|im_start|>user +안녕하세요! 오늘 날씨가 어때요?<|im_end|> +<|im_start|>assistant +``` + +The model generates until it produces `<|im_end|>` (token ID 131073). + +> **Important:** Always include a system prompt. Without it, the model may default to Korean regardless of input language. + +## How to Use + +### With `transformers` + +```python +from transformers import AutoTokenizer, AutoModelForCausalLM +import torch + +model_id = "mkd-hossain/keural-dpo-5500" + +tokenizer = AutoTokenizer.from_pretrained(model_id) +model = AutoModelForCausalLM.from_pretrained( + model_id, + torch_dtype=torch.bfloat16, + device_map="auto", +) + +messages = [ + { + "role": "system", + "content": ( + "You are a helpful bilingual Korean-English assistant. " + "Always respond in the same language as the user's message. " + "If the user writes in English, respond in English. " + "If the user writes in Korean, respond in Korean." + ) + }, + {"role": "user", "content": "파이썬에서 리스트를 정렬하는 방법을 알려주세요."}, +] + +text = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True) +inputs = tokenizer(text, return_tensors="pt").to(model.device) + +with torch.no_grad(): + output = model.generate( + **inputs, + max_new_tokens=512, + temperature=0.7, + top_p=0.9, + top_k=50, + repetition_penalty=1.1, + no_repeat_ngram_size=8, + do_sample=True, + eos_token_id=131073, # <|im_end|> + ) + +response = tokenizer.decode(output[0][inputs.input_ids.shape[1]:], skip_special_tokens=False) +response = response.split("<|im_end|>")[0].strip() +print(response) +``` + +### With vLLM (recommended for serving) + +```bash +pip install vllm + +python -m vllm.entrypoints.openai.api_server \ + --model mkd-hossain/keural-dpo-5500 \ + --tokenizer mkd-hossain/keural-dpo-5500 \ + --dtype bfloat16 \ + --max-model-len 4096 \ + --tensor-parallel-size 1 +``` + +Call the OpenAI-compatible endpoint: + +```python +from openai import OpenAI + +client = OpenAI(base_url="http://localhost:8000/v1", api_key="none") + +response = client.chat.completions.create( + model="mkd-hossain/keural-dpo-5500", + messages=[ + { + "role": "system", + "content": "You are a helpful bilingual assistant. Respond in the same language as the user." + }, + {"role": "user", "content": "What is the capital of South Korea?"}, + ], + max_tokens=512, + temperature=0.7, +) +print(response.choices[0].message.content) +``` + +### Multi-GPU serving (2× GPU) + +```bash +python -m vllm.entrypoints.openai.api_server \ + --model mkd-hossain/keural-dpo-5500 \ + --dtype bfloat16 \ + --max-model-len 4096 \ + --tensor-parallel-size 2 +``` + +### Manual ChatML prompt (without `apply_chat_template`) + +```python +prompt = ( + "<|im_start|>system\n" + "You are a helpful bilingual Korean-English assistant. " + "Always respond in the same language as the user.\n" + "<|im_end|>\n" + "<|im_start|>user\n" + "Tell me about Seoul.<|im_end|>\n" + "<|im_start|>assistant\n" +) +``` + +## Special Tokens + +| Token | ID | Purpose | +|---|---|---| +| `<|im_start|>` | 131072 | Marks the start of each conversation turn | +| `<|im_end|>` | 131073 | Marks the end of each turn / generation stop token | +| `` | 1 | Beginning of sequence | +| `` | 2 | End of sequence | +| `` | 0 | Padding token | + +> **Critical:** Always set `eos_token_id=131073` (`<|im_end|>`) when generating. Using `eos_token_id=2` will cause generation to not stop correctly. + +## Recommended Generation Settings + +```python +# For conversational / creative tasks +generation_config = { + "max_new_tokens": 512, + "temperature": 0.7, + "top_p": 0.9, + "top_k": 50, + "repetition_penalty": 1.1, + "no_repeat_ngram_size": 8, + "do_sample": True, + "eos_token_id": 131073, +} + +# For factual / deterministic tasks +generation_config = { + "max_new_tokens": 512, + "temperature": 0.1, + "repetition_penalty": 1.1, + "no_repeat_ngram_size": 8, + "do_sample": False, + "eos_token_id": 131073, +} +``` + +## DPO Dataset + +Training used the `keural-dpo-raw` dataset — 440,627 chosen/rejected preference pairs in ChatML format, covering: +- General conversation (Korean and English) +- Question answering +- Instruction following +- Knowledge and reasoning tasks + +## Comparison to Previous Checkpoints + +| Checkpoint | Stage | Key Difference | +|---|---|---| +| mkd-hossain/keural-pretrained | Pretraining (120k steps) | Raw base model, no instruction tuning | +| mkd-hossain/keural-sft-18k | SFT (18k steps) | Instruction following, ChatML format | +| mkd-hossain/keural-dpo-3500 | DPO 50% | Early alignment, margins emerging | +| **mkd-hossain/keural-dpo-5500** | **DPO 79%** | **Stronger alignment, consistent margins** | + +## Limitations + +- This is a **late-training checkpoint** (step 5,500 of 6,927 — 79% of 1 epoch). A full-epoch checkpoint will be released when training completes. +- Maximum context is 4,096 tokens. Inputs longer than this will be truncated. +- The pretraining corpus is Korean-dominant. Always include a system prompt for correct bilingual behavior. +- Not safety-aligned — do not deploy in production without additional safety fine-tuning. +- DPO margins are small (0.001–0.002) due to the large model size and low LR — this is normal for 14B+ models. + +## License + +Apache 2.0 diff --git a/config.json b/config.json new file mode 100644 index 0000000..0ed33d6 --- /dev/null +++ b/config.json @@ -0,0 +1,32 @@ +{ + "architectures": [ + "KeuralMoECausalLM" + ], + "model_type": "keural", + "vocab_size": 131074, + "hidden_size": 4096, + "intermediate_size": 5632, + "num_hidden_layers": 24, + "num_attention_heads": 32, + "num_key_value_heads": 8, + "head_dim": 128, + "num_local_experts": 8, + "num_experts_per_tok": 2, + "max_position_embeddings": 4096, + "rope_theta": 500000.0, + "rms_norm_eps": 1e-05, + "sliding_window": 512, + "hidden_act": "silu", + "initializer_range": 0.02, + "use_cache": true, + "tie_word_embeddings": false, + "torch_dtype": "bfloat16", + "transformers_version": "4.40.0", + "output_router_logits": false, + "router_aux_loss_coef": 0.001, + "keural_step": 5500, + "auto_map": { + "AutoConfig": "configuration_keural.KeuralConfig", + "AutoModelForCausalLM": "modeling_keural.KeuralMoECausalLM" + } +} \ No newline at end of file diff --git a/configuration_keural.py b/configuration_keural.py new file mode 100644 index 0000000..11a9bbd --- /dev/null +++ b/configuration_keural.py @@ -0,0 +1,13 @@ +""" +KeuralConfig — custom configuration class for Keural MoE. +Registered as model_type "keural" so HuggingFace AutoConfig can resolve it. +""" + +from transformers import MixtralConfig + + +class KeuralConfig(MixtralConfig): + model_type = "keural" + + def __init__(self, **kwargs): + super().__init__(**kwargs) diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..f27305e --- /dev/null +++ b/generation_config.json @@ -0,0 +1,12 @@ +{ + "bos_token_id": 1, + "eos_token_id": 2, + "pad_token_id": 0, + "max_new_tokens": 512, + "temperature": 0.7, + "top_p": 0.9, + "top_k": 50, + "repetition_penalty": 1.1, + "do_sample": true, + "transformers_version": "4.40.0" +} \ No newline at end of file diff --git a/model-00001-of-00007.safetensors b/model-00001-of-00007.safetensors new file mode 100644 index 0000000..4a4ec68 --- /dev/null +++ b/model-00001-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3104e300cda5d354f48b7d32a8200f7056895ad3c92747233983440f18493faf +size 4823806664 diff --git a/model-00002-of-00007.safetensors b/model-00002-of-00007.safetensors new file mode 100644 index 0000000..972ddce --- /dev/null +++ b/model-00002-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f6be94acc7bd003cb9aa02bccf41d35e8c6d67a8bf8145eb772e1f81b061b3bd +size 4811210208 diff --git a/model-00003-of-00007.safetensors b/model-00003-of-00007.safetensors new file mode 100644 index 0000000..92e37fa --- /dev/null +++ b/model-00003-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:16bd6e372b41ee489cf4e01b0e5d0e7408f32a1955ee2c2bed203dfe03721a6d +size 4811210248 diff --git a/model-00004-of-00007.safetensors b/model-00004-of-00007.safetensors new file mode 100644 index 0000000..9a253ed --- /dev/null +++ b/model-00004-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0b01c279ab30b9c425f2ffb9c487147ff05cc4abb82c6dbedafa26045da46b4e +size 4811210336 diff --git a/model-00005-of-00007.safetensors b/model-00005-of-00007.safetensors new file mode 100644 index 0000000..e88b3ed --- /dev/null +++ b/model-00005-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1d5d4721eece97213f5a836e6bc8fd55c8d4e2ef7a2e3eccb4537bc7e2809574 +size 4811210336 diff --git a/model-00006-of-00007.safetensors b/model-00006-of-00007.safetensors new file mode 100644 index 0000000..ddafbfa --- /dev/null +++ b/model-00006-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7e07978a3a9ec4395e2fe3b7ab41426d8918bbb144115ead1c6a3c511e2a0b47 +size 4811210336 diff --git a/model-00007-of-00007.safetensors b/model-00007-of-00007.safetensors new file mode 100644 index 0000000..d59b5ef --- /dev/null +++ b/model-00007-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0d0117b94f5b5d9f857e899b75743a0ee2bf5486640421b5d3f1d7909654cc33 +size 1858103664 diff --git a/model.safetensors.index.json b/model.safetensors.index.json new file mode 100644 index 0000000..f6ee7fb --- /dev/null +++ b/model.safetensors.index.json @@ -0,0 +1,754 @@ +{ + "metadata": { + "total_size": 30737866752 + }, + "weight_map": { + "model.embed_tokens.weight": "model-00001-of-00007.safetensors", + "model.layers.0.input_layernorm.weight": "model-00001-of-00007.safetensors", + "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00007.safetensors", + "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.0.block_sparse_moe.gate.weight": "model-00001-of-00007.safetensors", + "model.layers.0.block_sparse_moe.experts.0.w1.weight": "model-00001-of-00007.safetensors", + "model.layers.0.block_sparse_moe.experts.0.w3.weight": "model-00001-of-00007.safetensors", + "model.layers.0.block_sparse_moe.experts.0.w2.weight": "model-00001-of-00007.safetensors", + "model.layers.0.block_sparse_moe.experts.1.w1.weight": "model-00001-of-00007.safetensors", + "model.layers.0.block_sparse_moe.experts.1.w3.weight": "model-00001-of-00007.safetensors", + "model.layers.0.block_sparse_moe.experts.1.w2.weight": "model-00001-of-00007.safetensors", + "model.layers.0.block_sparse_moe.experts.2.w1.weight": "model-00001-of-00007.safetensors", + "model.layers.0.block_sparse_moe.experts.2.w3.weight": "model-00001-of-00007.safetensors", + "model.layers.0.block_sparse_moe.experts.2.w2.weight": "model-00001-of-00007.safetensors", + "model.layers.0.block_sparse_moe.experts.3.w1.weight": "model-00001-of-00007.safetensors", + "model.layers.0.block_sparse_moe.experts.3.w3.weight": "model-00001-of-00007.safetensors", + "model.layers.0.block_sparse_moe.experts.3.w2.weight": "model-00001-of-00007.safetensors", + "model.layers.0.block_sparse_moe.experts.4.w1.weight": "model-00001-of-00007.safetensors", + "model.layers.0.block_sparse_moe.experts.4.w3.weight": "model-00001-of-00007.safetensors", + "model.layers.0.block_sparse_moe.experts.4.w2.weight": "model-00001-of-00007.safetensors", + "model.layers.0.block_sparse_moe.experts.5.w1.weight": "model-00001-of-00007.safetensors", + "model.layers.0.block_sparse_moe.experts.5.w3.weight": "model-00001-of-00007.safetensors", + "model.layers.0.block_sparse_moe.experts.5.w2.weight": "model-00001-of-00007.safetensors", + "model.layers.0.block_sparse_moe.experts.6.w1.weight": "model-00001-of-00007.safetensors", + "model.layers.0.block_sparse_moe.experts.6.w3.weight": "model-00001-of-00007.safetensors", + "model.layers.0.block_sparse_moe.experts.6.w2.weight": "model-00001-of-00007.safetensors", + "model.layers.0.block_sparse_moe.experts.7.w1.weight": "model-00001-of-00007.safetensors", + "model.layers.0.block_sparse_moe.experts.7.w3.weight": "model-00001-of-00007.safetensors", + "model.layers.0.block_sparse_moe.experts.7.w2.weight": "model-00001-of-00007.safetensors", + "model.layers.1.input_layernorm.weight": "model-00001-of-00007.safetensors", + "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00007.safetensors", + "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.block_sparse_moe.gate.weight": "model-00001-of-00007.safetensors", + "model.layers.1.block_sparse_moe.experts.0.w1.weight": "model-00001-of-00007.safetensors", + "model.layers.1.block_sparse_moe.experts.0.w3.weight": "model-00001-of-00007.safetensors", + "model.layers.1.block_sparse_moe.experts.0.w2.weight": "model-00001-of-00007.safetensors", + "model.layers.1.block_sparse_moe.experts.1.w1.weight": "model-00001-of-00007.safetensors", + "model.layers.1.block_sparse_moe.experts.1.w3.weight": "model-00001-of-00007.safetensors", + "model.layers.1.block_sparse_moe.experts.1.w2.weight": "model-00001-of-00007.safetensors", + "model.layers.1.block_sparse_moe.experts.2.w1.weight": "model-00001-of-00007.safetensors", + "model.layers.1.block_sparse_moe.experts.2.w3.weight": "model-00001-of-00007.safetensors", + "model.layers.1.block_sparse_moe.experts.2.w2.weight": "model-00001-of-00007.safetensors", + "model.layers.1.block_sparse_moe.experts.3.w1.weight": "model-00001-of-00007.safetensors", + "model.layers.1.block_sparse_moe.experts.3.w3.weight": "model-00001-of-00007.safetensors", + "model.layers.1.block_sparse_moe.experts.3.w2.weight": "model-00001-of-00007.safetensors", + "model.layers.1.block_sparse_moe.experts.4.w1.weight": "model-00001-of-00007.safetensors", + "model.layers.1.block_sparse_moe.experts.4.w3.weight": "model-00001-of-00007.safetensors", + "model.layers.1.block_sparse_moe.experts.4.w2.weight": "model-00001-of-00007.safetensors", + "model.layers.1.block_sparse_moe.experts.5.w1.weight": "model-00001-of-00007.safetensors", + "model.layers.1.block_sparse_moe.experts.5.w3.weight": "model-00001-of-00007.safetensors", + "model.layers.1.block_sparse_moe.experts.5.w2.weight": "model-00001-of-00007.safetensors", + "model.layers.1.block_sparse_moe.experts.6.w1.weight": "model-00001-of-00007.safetensors", + "model.layers.1.block_sparse_moe.experts.6.w3.weight": "model-00001-of-00007.safetensors", + "model.layers.1.block_sparse_moe.experts.6.w2.weight": "model-00001-of-00007.safetensors", + "model.layers.1.block_sparse_moe.experts.7.w1.weight": "model-00001-of-00007.safetensors", + "model.layers.1.block_sparse_moe.experts.7.w3.weight": "model-00001-of-00007.safetensors", + "model.layers.1.block_sparse_moe.experts.7.w2.weight": "model-00001-of-00007.safetensors", + "model.layers.2.input_layernorm.weight": "model-00001-of-00007.safetensors", + "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00007.safetensors", + "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.2.block_sparse_moe.gate.weight": "model-00001-of-00007.safetensors", + "model.layers.2.block_sparse_moe.experts.0.w1.weight": "model-00001-of-00007.safetensors", + "model.layers.2.block_sparse_moe.experts.0.w3.weight": "model-00001-of-00007.safetensors", + "model.layers.2.block_sparse_moe.experts.0.w2.weight": "model-00001-of-00007.safetensors", + "model.layers.2.block_sparse_moe.experts.1.w1.weight": "model-00001-of-00007.safetensors", + "model.layers.2.block_sparse_moe.experts.1.w3.weight": "model-00001-of-00007.safetensors", + "model.layers.2.block_sparse_moe.experts.1.w2.weight": "model-00001-of-00007.safetensors", + "model.layers.2.block_sparse_moe.experts.2.w1.weight": "model-00001-of-00007.safetensors", + "model.layers.2.block_sparse_moe.experts.2.w3.weight": "model-00001-of-00007.safetensors", + "model.layers.2.block_sparse_moe.experts.2.w2.weight": "model-00001-of-00007.safetensors", + "model.layers.2.block_sparse_moe.experts.3.w1.weight": "model-00001-of-00007.safetensors", + "model.layers.2.block_sparse_moe.experts.3.w3.weight": "model-00001-of-00007.safetensors", + "model.layers.2.block_sparse_moe.experts.3.w2.weight": "model-00001-of-00007.safetensors", + "model.layers.2.block_sparse_moe.experts.4.w1.weight": "model-00001-of-00007.safetensors", + "model.layers.2.block_sparse_moe.experts.4.w3.weight": "model-00001-of-00007.safetensors", + "model.layers.2.block_sparse_moe.experts.4.w2.weight": "model-00001-of-00007.safetensors", + "model.layers.2.block_sparse_moe.experts.5.w1.weight": "model-00001-of-00007.safetensors", + "model.layers.2.block_sparse_moe.experts.5.w3.weight": "model-00001-of-00007.safetensors", + "model.layers.2.block_sparse_moe.experts.5.w2.weight": "model-00001-of-00007.safetensors", + "model.layers.2.block_sparse_moe.experts.6.w1.weight": "model-00001-of-00007.safetensors", + "model.layers.2.block_sparse_moe.experts.6.w3.weight": "model-00001-of-00007.safetensors", + "model.layers.2.block_sparse_moe.experts.6.w2.weight": "model-00001-of-00007.safetensors", + "model.layers.2.block_sparse_moe.experts.7.w1.weight": "model-00001-of-00007.safetensors", + "model.layers.2.block_sparse_moe.experts.7.w3.weight": "model-00001-of-00007.safetensors", + "model.layers.2.block_sparse_moe.experts.7.w2.weight": "model-00001-of-00007.safetensors", + "model.layers.3.input_layernorm.weight": "model-00001-of-00007.safetensors", + "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00007.safetensors", + "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.3.block_sparse_moe.gate.weight": "model-00001-of-00007.safetensors", + "model.layers.3.block_sparse_moe.experts.0.w1.weight": "model-00001-of-00007.safetensors", + "model.layers.3.block_sparse_moe.experts.0.w3.weight": "model-00001-of-00007.safetensors", + "model.layers.3.block_sparse_moe.experts.0.w2.weight": "model-00002-of-00007.safetensors", + "model.layers.3.block_sparse_moe.experts.1.w1.weight": "model-00002-of-00007.safetensors", + "model.layers.3.block_sparse_moe.experts.1.w3.weight": "model-00002-of-00007.safetensors", + "model.layers.3.block_sparse_moe.experts.1.w2.weight": "model-00002-of-00007.safetensors", + "model.layers.3.block_sparse_moe.experts.2.w1.weight": "model-00002-of-00007.safetensors", + "model.layers.3.block_sparse_moe.experts.2.w3.weight": "model-00002-of-00007.safetensors", + "model.layers.3.block_sparse_moe.experts.2.w2.weight": "model-00002-of-00007.safetensors", + "model.layers.3.block_sparse_moe.experts.3.w1.weight": "model-00002-of-00007.safetensors", + "model.layers.3.block_sparse_moe.experts.3.w3.weight": "model-00002-of-00007.safetensors", + "model.layers.3.block_sparse_moe.experts.3.w2.weight": "model-00002-of-00007.safetensors", + "model.layers.3.block_sparse_moe.experts.4.w1.weight": "model-00002-of-00007.safetensors", + "model.layers.3.block_sparse_moe.experts.4.w3.weight": "model-00002-of-00007.safetensors", + "model.layers.3.block_sparse_moe.experts.4.w2.weight": "model-00002-of-00007.safetensors", + "model.layers.3.block_sparse_moe.experts.5.w1.weight": "model-00002-of-00007.safetensors", + "model.layers.3.block_sparse_moe.experts.5.w3.weight": "model-00002-of-00007.safetensors", + "model.layers.3.block_sparse_moe.experts.5.w2.weight": "model-00002-of-00007.safetensors", + "model.layers.3.block_sparse_moe.experts.6.w1.weight": "model-00002-of-00007.safetensors", + "model.layers.3.block_sparse_moe.experts.6.w3.weight": "model-00002-of-00007.safetensors", + "model.layers.3.block_sparse_moe.experts.6.w2.weight": "model-00002-of-00007.safetensors", + "model.layers.3.block_sparse_moe.experts.7.w1.weight": "model-00002-of-00007.safetensors", + "model.layers.3.block_sparse_moe.experts.7.w3.weight": "model-00002-of-00007.safetensors", + "model.layers.3.block_sparse_moe.experts.7.w2.weight": "model-00002-of-00007.safetensors", + "model.layers.4.input_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.4.post_attention_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.4.self_attn.q_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.4.self_attn.k_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.4.self_attn.v_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.4.self_attn.o_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.4.block_sparse_moe.gate.weight": "model-00002-of-00007.safetensors", + "model.layers.4.block_sparse_moe.experts.0.w1.weight": "model-00002-of-00007.safetensors", + "model.layers.4.block_sparse_moe.experts.0.w3.weight": "model-00002-of-00007.safetensors", + "model.layers.4.block_sparse_moe.experts.0.w2.weight": "model-00002-of-00007.safetensors", + "model.layers.4.block_sparse_moe.experts.1.w1.weight": "model-00002-of-00007.safetensors", + "model.layers.4.block_sparse_moe.experts.1.w3.weight": "model-00002-of-00007.safetensors", + "model.layers.4.block_sparse_moe.experts.1.w2.weight": "model-00002-of-00007.safetensors", + "model.layers.4.block_sparse_moe.experts.2.w1.weight": "model-00002-of-00007.safetensors", + "model.layers.4.block_sparse_moe.experts.2.w3.weight": "model-00002-of-00007.safetensors", + "model.layers.4.block_sparse_moe.experts.2.w2.weight": "model-00002-of-00007.safetensors", + "model.layers.4.block_sparse_moe.experts.3.w1.weight": "model-00002-of-00007.safetensors", + "model.layers.4.block_sparse_moe.experts.3.w3.weight": "model-00002-of-00007.safetensors", + "model.layers.4.block_sparse_moe.experts.3.w2.weight": "model-00002-of-00007.safetensors", + "model.layers.4.block_sparse_moe.experts.4.w1.weight": "model-00002-of-00007.safetensors", + "model.layers.4.block_sparse_moe.experts.4.w3.weight": "model-00002-of-00007.safetensors", + "model.layers.4.block_sparse_moe.experts.4.w2.weight": "model-00002-of-00007.safetensors", + "model.layers.4.block_sparse_moe.experts.5.w1.weight": "model-00002-of-00007.safetensors", + "model.layers.4.block_sparse_moe.experts.5.w3.weight": "model-00002-of-00007.safetensors", + "model.layers.4.block_sparse_moe.experts.5.w2.weight": "model-00002-of-00007.safetensors", + "model.layers.4.block_sparse_moe.experts.6.w1.weight": "model-00002-of-00007.safetensors", + "model.layers.4.block_sparse_moe.experts.6.w3.weight": "model-00002-of-00007.safetensors", + "model.layers.4.block_sparse_moe.experts.6.w2.weight": "model-00002-of-00007.safetensors", + "model.layers.4.block_sparse_moe.experts.7.w1.weight": "model-00002-of-00007.safetensors", + "model.layers.4.block_sparse_moe.experts.7.w3.weight": "model-00002-of-00007.safetensors", + "model.layers.4.block_sparse_moe.experts.7.w2.weight": "model-00002-of-00007.safetensors", + "model.layers.5.input_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.5.post_attention_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.5.self_attn.q_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.self_attn.k_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.self_attn.v_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.self_attn.o_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.block_sparse_moe.gate.weight": "model-00002-of-00007.safetensors", + "model.layers.5.block_sparse_moe.experts.0.w1.weight": "model-00002-of-00007.safetensors", + "model.layers.5.block_sparse_moe.experts.0.w3.weight": "model-00002-of-00007.safetensors", + "model.layers.5.block_sparse_moe.experts.0.w2.weight": "model-00002-of-00007.safetensors", + "model.layers.5.block_sparse_moe.experts.1.w1.weight": "model-00002-of-00007.safetensors", + "model.layers.5.block_sparse_moe.experts.1.w3.weight": "model-00002-of-00007.safetensors", + "model.layers.5.block_sparse_moe.experts.1.w2.weight": "model-00002-of-00007.safetensors", + "model.layers.5.block_sparse_moe.experts.2.w1.weight": "model-00002-of-00007.safetensors", + "model.layers.5.block_sparse_moe.experts.2.w3.weight": "model-00002-of-00007.safetensors", + "model.layers.5.block_sparse_moe.experts.2.w2.weight": "model-00002-of-00007.safetensors", + "model.layers.5.block_sparse_moe.experts.3.w1.weight": "model-00002-of-00007.safetensors", + "model.layers.5.block_sparse_moe.experts.3.w3.weight": "model-00002-of-00007.safetensors", + "model.layers.5.block_sparse_moe.experts.3.w2.weight": "model-00002-of-00007.safetensors", + "model.layers.5.block_sparse_moe.experts.4.w1.weight": "model-00002-of-00007.safetensors", + "model.layers.5.block_sparse_moe.experts.4.w3.weight": "model-00002-of-00007.safetensors", + "model.layers.5.block_sparse_moe.experts.4.w2.weight": "model-00002-of-00007.safetensors", + "model.layers.5.block_sparse_moe.experts.5.w1.weight": "model-00002-of-00007.safetensors", + "model.layers.5.block_sparse_moe.experts.5.w3.weight": "model-00002-of-00007.safetensors", + "model.layers.5.block_sparse_moe.experts.5.w2.weight": "model-00002-of-00007.safetensors", + "model.layers.5.block_sparse_moe.experts.6.w1.weight": "model-00002-of-00007.safetensors", + "model.layers.5.block_sparse_moe.experts.6.w3.weight": "model-00002-of-00007.safetensors", + "model.layers.5.block_sparse_moe.experts.6.w2.weight": "model-00002-of-00007.safetensors", + "model.layers.5.block_sparse_moe.experts.7.w1.weight": "model-00002-of-00007.safetensors", + "model.layers.5.block_sparse_moe.experts.7.w3.weight": "model-00002-of-00007.safetensors", + "model.layers.5.block_sparse_moe.experts.7.w2.weight": "model-00002-of-00007.safetensors", + "model.layers.6.input_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.6.post_attention_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.6.self_attn.q_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.self_attn.k_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.self_attn.v_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.self_attn.o_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.block_sparse_moe.gate.weight": "model-00002-of-00007.safetensors", + "model.layers.6.block_sparse_moe.experts.0.w1.weight": "model-00002-of-00007.safetensors", + "model.layers.6.block_sparse_moe.experts.0.w3.weight": "model-00002-of-00007.safetensors", + "model.layers.6.block_sparse_moe.experts.0.w2.weight": "model-00002-of-00007.safetensors", + "model.layers.6.block_sparse_moe.experts.1.w1.weight": "model-00002-of-00007.safetensors", + "model.layers.6.block_sparse_moe.experts.1.w3.weight": "model-00002-of-00007.safetensors", + "model.layers.6.block_sparse_moe.experts.1.w2.weight": "model-00002-of-00007.safetensors", + "model.layers.6.block_sparse_moe.experts.2.w1.weight": "model-00002-of-00007.safetensors", + "model.layers.6.block_sparse_moe.experts.2.w3.weight": "model-00002-of-00007.safetensors", + "model.layers.6.block_sparse_moe.experts.2.w2.weight": "model-00002-of-00007.safetensors", + "model.layers.6.block_sparse_moe.experts.3.w1.weight": "model-00002-of-00007.safetensors", + "model.layers.6.block_sparse_moe.experts.3.w3.weight": "model-00002-of-00007.safetensors", + "model.layers.6.block_sparse_moe.experts.3.w2.weight": "model-00002-of-00007.safetensors", + "model.layers.6.block_sparse_moe.experts.4.w1.weight": "model-00002-of-00007.safetensors", + "model.layers.6.block_sparse_moe.experts.4.w3.weight": "model-00002-of-00007.safetensors", + "model.layers.6.block_sparse_moe.experts.4.w2.weight": "model-00002-of-00007.safetensors", + "model.layers.6.block_sparse_moe.experts.5.w1.weight": "model-00002-of-00007.safetensors", + "model.layers.6.block_sparse_moe.experts.5.w3.weight": "model-00002-of-00007.safetensors", + "model.layers.6.block_sparse_moe.experts.5.w2.weight": "model-00002-of-00007.safetensors", + "model.layers.6.block_sparse_moe.experts.6.w1.weight": "model-00002-of-00007.safetensors", + "model.layers.6.block_sparse_moe.experts.6.w3.weight": "model-00002-of-00007.safetensors", + "model.layers.6.block_sparse_moe.experts.6.w2.weight": "model-00002-of-00007.safetensors", + "model.layers.6.block_sparse_moe.experts.7.w1.weight": "model-00002-of-00007.safetensors", + "model.layers.6.block_sparse_moe.experts.7.w3.weight": "model-00002-of-00007.safetensors", + "model.layers.6.block_sparse_moe.experts.7.w2.weight": "model-00002-of-00007.safetensors", + "model.layers.7.input_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.7.self_attn.q_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.self_attn.k_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.self_attn.v_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.self_attn.o_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.block_sparse_moe.gate.weight": "model-00002-of-00007.safetensors", + "model.layers.7.block_sparse_moe.experts.0.w1.weight": "model-00002-of-00007.safetensors", + "model.layers.7.block_sparse_moe.experts.0.w3.weight": "model-00002-of-00007.safetensors", + "model.layers.7.block_sparse_moe.experts.0.w2.weight": "model-00002-of-00007.safetensors", + "model.layers.7.block_sparse_moe.experts.1.w1.weight": "model-00003-of-00007.safetensors", + "model.layers.7.block_sparse_moe.experts.1.w3.weight": "model-00003-of-00007.safetensors", + "model.layers.7.block_sparse_moe.experts.1.w2.weight": "model-00003-of-00007.safetensors", + "model.layers.7.block_sparse_moe.experts.2.w1.weight": "model-00003-of-00007.safetensors", + "model.layers.7.block_sparse_moe.experts.2.w3.weight": "model-00003-of-00007.safetensors", + "model.layers.7.block_sparse_moe.experts.2.w2.weight": "model-00003-of-00007.safetensors", + "model.layers.7.block_sparse_moe.experts.3.w1.weight": "model-00003-of-00007.safetensors", + "model.layers.7.block_sparse_moe.experts.3.w3.weight": "model-00003-of-00007.safetensors", + "model.layers.7.block_sparse_moe.experts.3.w2.weight": "model-00003-of-00007.safetensors", + "model.layers.7.block_sparse_moe.experts.4.w1.weight": "model-00003-of-00007.safetensors", + "model.layers.7.block_sparse_moe.experts.4.w3.weight": "model-00003-of-00007.safetensors", + "model.layers.7.block_sparse_moe.experts.4.w2.weight": "model-00003-of-00007.safetensors", + "model.layers.7.block_sparse_moe.experts.5.w1.weight": "model-00003-of-00007.safetensors", + "model.layers.7.block_sparse_moe.experts.5.w3.weight": "model-00003-of-00007.safetensors", + "model.layers.7.block_sparse_moe.experts.5.w2.weight": "model-00003-of-00007.safetensors", + "model.layers.7.block_sparse_moe.experts.6.w1.weight": "model-00003-of-00007.safetensors", + "model.layers.7.block_sparse_moe.experts.6.w3.weight": "model-00003-of-00007.safetensors", + "model.layers.7.block_sparse_moe.experts.6.w2.weight": "model-00003-of-00007.safetensors", + "model.layers.7.block_sparse_moe.experts.7.w1.weight": "model-00003-of-00007.safetensors", + "model.layers.7.block_sparse_moe.experts.7.w3.weight": "model-00003-of-00007.safetensors", + "model.layers.7.block_sparse_moe.experts.7.w2.weight": "model-00003-of-00007.safetensors", + "model.layers.8.input_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.8.post_attention_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.8.self_attn.q_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.8.self_attn.k_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.8.self_attn.v_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.8.self_attn.o_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.8.block_sparse_moe.gate.weight": "model-00003-of-00007.safetensors", + "model.layers.8.block_sparse_moe.experts.0.w1.weight": "model-00003-of-00007.safetensors", + "model.layers.8.block_sparse_moe.experts.0.w3.weight": "model-00003-of-00007.safetensors", + "model.layers.8.block_sparse_moe.experts.0.w2.weight": "model-00003-of-00007.safetensors", + "model.layers.8.block_sparse_moe.experts.1.w1.weight": "model-00003-of-00007.safetensors", + "model.layers.8.block_sparse_moe.experts.1.w3.weight": "model-00003-of-00007.safetensors", + "model.layers.8.block_sparse_moe.experts.1.w2.weight": "model-00003-of-00007.safetensors", + "model.layers.8.block_sparse_moe.experts.2.w1.weight": "model-00003-of-00007.safetensors", + "model.layers.8.block_sparse_moe.experts.2.w3.weight": "model-00003-of-00007.safetensors", + "model.layers.8.block_sparse_moe.experts.2.w2.weight": "model-00003-of-00007.safetensors", + "model.layers.8.block_sparse_moe.experts.3.w1.weight": "model-00003-of-00007.safetensors", + "model.layers.8.block_sparse_moe.experts.3.w3.weight": "model-00003-of-00007.safetensors", + "model.layers.8.block_sparse_moe.experts.3.w2.weight": "model-00003-of-00007.safetensors", + "model.layers.8.block_sparse_moe.experts.4.w1.weight": "model-00003-of-00007.safetensors", + "model.layers.8.block_sparse_moe.experts.4.w3.weight": "model-00003-of-00007.safetensors", + "model.layers.8.block_sparse_moe.experts.4.w2.weight": "model-00003-of-00007.safetensors", + "model.layers.8.block_sparse_moe.experts.5.w1.weight": "model-00003-of-00007.safetensors", + "model.layers.8.block_sparse_moe.experts.5.w3.weight": "model-00003-of-00007.safetensors", + "model.layers.8.block_sparse_moe.experts.5.w2.weight": "model-00003-of-00007.safetensors", + "model.layers.8.block_sparse_moe.experts.6.w1.weight": "model-00003-of-00007.safetensors", + "model.layers.8.block_sparse_moe.experts.6.w3.weight": "model-00003-of-00007.safetensors", + "model.layers.8.block_sparse_moe.experts.6.w2.weight": "model-00003-of-00007.safetensors", + "model.layers.8.block_sparse_moe.experts.7.w1.weight": "model-00003-of-00007.safetensors", + "model.layers.8.block_sparse_moe.experts.7.w3.weight": "model-00003-of-00007.safetensors", + "model.layers.8.block_sparse_moe.experts.7.w2.weight": "model-00003-of-00007.safetensors", + "model.layers.9.input_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.9.post_attention_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.9.self_attn.q_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.9.self_attn.k_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.9.self_attn.v_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.9.self_attn.o_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.9.block_sparse_moe.gate.weight": "model-00003-of-00007.safetensors", + "model.layers.9.block_sparse_moe.experts.0.w1.weight": "model-00003-of-00007.safetensors", + "model.layers.9.block_sparse_moe.experts.0.w3.weight": "model-00003-of-00007.safetensors", + "model.layers.9.block_sparse_moe.experts.0.w2.weight": "model-00003-of-00007.safetensors", + "model.layers.9.block_sparse_moe.experts.1.w1.weight": "model-00003-of-00007.safetensors", + "model.layers.9.block_sparse_moe.experts.1.w3.weight": "model-00003-of-00007.safetensors", + "model.layers.9.block_sparse_moe.experts.1.w2.weight": "model-00003-of-00007.safetensors", + "model.layers.9.block_sparse_moe.experts.2.w1.weight": "model-00003-of-00007.safetensors", + "model.layers.9.block_sparse_moe.experts.2.w3.weight": "model-00003-of-00007.safetensors", + "model.layers.9.block_sparse_moe.experts.2.w2.weight": "model-00003-of-00007.safetensors", + "model.layers.9.block_sparse_moe.experts.3.w1.weight": "model-00003-of-00007.safetensors", + "model.layers.9.block_sparse_moe.experts.3.w3.weight": "model-00003-of-00007.safetensors", + "model.layers.9.block_sparse_moe.experts.3.w2.weight": "model-00003-of-00007.safetensors", + "model.layers.9.block_sparse_moe.experts.4.w1.weight": "model-00003-of-00007.safetensors", + "model.layers.9.block_sparse_moe.experts.4.w3.weight": "model-00003-of-00007.safetensors", + "model.layers.9.block_sparse_moe.experts.4.w2.weight": "model-00003-of-00007.safetensors", + "model.layers.9.block_sparse_moe.experts.5.w1.weight": "model-00003-of-00007.safetensors", + "model.layers.9.block_sparse_moe.experts.5.w3.weight": "model-00003-of-00007.safetensors", + "model.layers.9.block_sparse_moe.experts.5.w2.weight": "model-00003-of-00007.safetensors", + "model.layers.9.block_sparse_moe.experts.6.w1.weight": "model-00003-of-00007.safetensors", + "model.layers.9.block_sparse_moe.experts.6.w3.weight": "model-00003-of-00007.safetensors", + "model.layers.9.block_sparse_moe.experts.6.w2.weight": "model-00003-of-00007.safetensors", + "model.layers.9.block_sparse_moe.experts.7.w1.weight": "model-00003-of-00007.safetensors", + "model.layers.9.block_sparse_moe.experts.7.w3.weight": "model-00003-of-00007.safetensors", + "model.layers.9.block_sparse_moe.experts.7.w2.weight": "model-00003-of-00007.safetensors", + "model.layers.10.input_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.10.post_attention_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.10.self_attn.q_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.10.self_attn.k_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.10.self_attn.v_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.10.self_attn.o_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.10.block_sparse_moe.gate.weight": "model-00003-of-00007.safetensors", + "model.layers.10.block_sparse_moe.experts.0.w1.weight": "model-00003-of-00007.safetensors", + "model.layers.10.block_sparse_moe.experts.0.w3.weight": "model-00003-of-00007.safetensors", + "model.layers.10.block_sparse_moe.experts.0.w2.weight": "model-00003-of-00007.safetensors", + "model.layers.10.block_sparse_moe.experts.1.w1.weight": "model-00003-of-00007.safetensors", + "model.layers.10.block_sparse_moe.experts.1.w3.weight": "model-00003-of-00007.safetensors", + "model.layers.10.block_sparse_moe.experts.1.w2.weight": "model-00003-of-00007.safetensors", + "model.layers.10.block_sparse_moe.experts.2.w1.weight": "model-00003-of-00007.safetensors", + "model.layers.10.block_sparse_moe.experts.2.w3.weight": "model-00003-of-00007.safetensors", + "model.layers.10.block_sparse_moe.experts.2.w2.weight": "model-00003-of-00007.safetensors", + "model.layers.10.block_sparse_moe.experts.3.w1.weight": "model-00003-of-00007.safetensors", + "model.layers.10.block_sparse_moe.experts.3.w3.weight": "model-00003-of-00007.safetensors", + "model.layers.10.block_sparse_moe.experts.3.w2.weight": "model-00003-of-00007.safetensors", + "model.layers.10.block_sparse_moe.experts.4.w1.weight": "model-00003-of-00007.safetensors", + "model.layers.10.block_sparse_moe.experts.4.w3.weight": "model-00003-of-00007.safetensors", + "model.layers.10.block_sparse_moe.experts.4.w2.weight": "model-00003-of-00007.safetensors", + "model.layers.10.block_sparse_moe.experts.5.w1.weight": "model-00003-of-00007.safetensors", + "model.layers.10.block_sparse_moe.experts.5.w3.weight": "model-00003-of-00007.safetensors", + "model.layers.10.block_sparse_moe.experts.5.w2.weight": "model-00003-of-00007.safetensors", + "model.layers.10.block_sparse_moe.experts.6.w1.weight": "model-00003-of-00007.safetensors", + "model.layers.10.block_sparse_moe.experts.6.w3.weight": "model-00003-of-00007.safetensors", + "model.layers.10.block_sparse_moe.experts.6.w2.weight": "model-00003-of-00007.safetensors", + "model.layers.10.block_sparse_moe.experts.7.w1.weight": "model-00003-of-00007.safetensors", + "model.layers.10.block_sparse_moe.experts.7.w3.weight": "model-00003-of-00007.safetensors", + "model.layers.10.block_sparse_moe.experts.7.w2.weight": "model-00003-of-00007.safetensors", + "model.layers.11.input_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.11.post_attention_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.11.self_attn.q_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.self_attn.k_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.self_attn.v_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.self_attn.o_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.block_sparse_moe.gate.weight": "model-00003-of-00007.safetensors", + "model.layers.11.block_sparse_moe.experts.0.w1.weight": "model-00003-of-00007.safetensors", + "model.layers.11.block_sparse_moe.experts.0.w3.weight": "model-00003-of-00007.safetensors", + "model.layers.11.block_sparse_moe.experts.0.w2.weight": "model-00003-of-00007.safetensors", + "model.layers.11.block_sparse_moe.experts.1.w1.weight": "model-00003-of-00007.safetensors", + "model.layers.11.block_sparse_moe.experts.1.w3.weight": "model-00004-of-00007.safetensors", + "model.layers.11.block_sparse_moe.experts.1.w2.weight": "model-00004-of-00007.safetensors", + "model.layers.11.block_sparse_moe.experts.2.w1.weight": "model-00004-of-00007.safetensors", + "model.layers.11.block_sparse_moe.experts.2.w3.weight": "model-00004-of-00007.safetensors", + "model.layers.11.block_sparse_moe.experts.2.w2.weight": "model-00004-of-00007.safetensors", + "model.layers.11.block_sparse_moe.experts.3.w1.weight": "model-00004-of-00007.safetensors", + "model.layers.11.block_sparse_moe.experts.3.w3.weight": "model-00004-of-00007.safetensors", + "model.layers.11.block_sparse_moe.experts.3.w2.weight": "model-00004-of-00007.safetensors", + "model.layers.11.block_sparse_moe.experts.4.w1.weight": "model-00004-of-00007.safetensors", + "model.layers.11.block_sparse_moe.experts.4.w3.weight": "model-00004-of-00007.safetensors", + "model.layers.11.block_sparse_moe.experts.4.w2.weight": "model-00004-of-00007.safetensors", + "model.layers.11.block_sparse_moe.experts.5.w1.weight": "model-00004-of-00007.safetensors", + "model.layers.11.block_sparse_moe.experts.5.w3.weight": "model-00004-of-00007.safetensors", + "model.layers.11.block_sparse_moe.experts.5.w2.weight": "model-00004-of-00007.safetensors", + "model.layers.11.block_sparse_moe.experts.6.w1.weight": "model-00004-of-00007.safetensors", + "model.layers.11.block_sparse_moe.experts.6.w3.weight": "model-00004-of-00007.safetensors", + "model.layers.11.block_sparse_moe.experts.6.w2.weight": "model-00004-of-00007.safetensors", + "model.layers.11.block_sparse_moe.experts.7.w1.weight": "model-00004-of-00007.safetensors", + "model.layers.11.block_sparse_moe.experts.7.w3.weight": "model-00004-of-00007.safetensors", + "model.layers.11.block_sparse_moe.experts.7.w2.weight": "model-00004-of-00007.safetensors", + "model.layers.12.input_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.12.post_attention_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.12.self_attn.q_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.12.self_attn.k_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.12.self_attn.v_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.12.self_attn.o_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.12.block_sparse_moe.gate.weight": "model-00004-of-00007.safetensors", + "model.layers.12.block_sparse_moe.experts.0.w1.weight": "model-00004-of-00007.safetensors", + "model.layers.12.block_sparse_moe.experts.0.w3.weight": "model-00004-of-00007.safetensors", + "model.layers.12.block_sparse_moe.experts.0.w2.weight": "model-00004-of-00007.safetensors", + "model.layers.12.block_sparse_moe.experts.1.w1.weight": "model-00004-of-00007.safetensors", + "model.layers.12.block_sparse_moe.experts.1.w3.weight": "model-00004-of-00007.safetensors", + "model.layers.12.block_sparse_moe.experts.1.w2.weight": "model-00004-of-00007.safetensors", + "model.layers.12.block_sparse_moe.experts.2.w1.weight": "model-00004-of-00007.safetensors", + "model.layers.12.block_sparse_moe.experts.2.w3.weight": "model-00004-of-00007.safetensors", + "model.layers.12.block_sparse_moe.experts.2.w2.weight": "model-00004-of-00007.safetensors", + "model.layers.12.block_sparse_moe.experts.3.w1.weight": "model-00004-of-00007.safetensors", + "model.layers.12.block_sparse_moe.experts.3.w3.weight": "model-00004-of-00007.safetensors", + "model.layers.12.block_sparse_moe.experts.3.w2.weight": "model-00004-of-00007.safetensors", + "model.layers.12.block_sparse_moe.experts.4.w1.weight": "model-00004-of-00007.safetensors", + "model.layers.12.block_sparse_moe.experts.4.w3.weight": "model-00004-of-00007.safetensors", + "model.layers.12.block_sparse_moe.experts.4.w2.weight": "model-00004-of-00007.safetensors", + "model.layers.12.block_sparse_moe.experts.5.w1.weight": "model-00004-of-00007.safetensors", + "model.layers.12.block_sparse_moe.experts.5.w3.weight": "model-00004-of-00007.safetensors", + "model.layers.12.block_sparse_moe.experts.5.w2.weight": "model-00004-of-00007.safetensors", + "model.layers.12.block_sparse_moe.experts.6.w1.weight": "model-00004-of-00007.safetensors", + "model.layers.12.block_sparse_moe.experts.6.w3.weight": "model-00004-of-00007.safetensors", + "model.layers.12.block_sparse_moe.experts.6.w2.weight": "model-00004-of-00007.safetensors", + "model.layers.12.block_sparse_moe.experts.7.w1.weight": "model-00004-of-00007.safetensors", + "model.layers.12.block_sparse_moe.experts.7.w3.weight": "model-00004-of-00007.safetensors", + "model.layers.12.block_sparse_moe.experts.7.w2.weight": "model-00004-of-00007.safetensors", + "model.layers.13.input_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.13.post_attention_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.13.self_attn.q_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.13.self_attn.k_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.13.self_attn.v_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.13.self_attn.o_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.13.block_sparse_moe.gate.weight": "model-00004-of-00007.safetensors", + "model.layers.13.block_sparse_moe.experts.0.w1.weight": "model-00004-of-00007.safetensors", + "model.layers.13.block_sparse_moe.experts.0.w3.weight": "model-00004-of-00007.safetensors", + "model.layers.13.block_sparse_moe.experts.0.w2.weight": "model-00004-of-00007.safetensors", + "model.layers.13.block_sparse_moe.experts.1.w1.weight": "model-00004-of-00007.safetensors", + "model.layers.13.block_sparse_moe.experts.1.w3.weight": "model-00004-of-00007.safetensors", + "model.layers.13.block_sparse_moe.experts.1.w2.weight": "model-00004-of-00007.safetensors", + "model.layers.13.block_sparse_moe.experts.2.w1.weight": "model-00004-of-00007.safetensors", + "model.layers.13.block_sparse_moe.experts.2.w3.weight": "model-00004-of-00007.safetensors", + "model.layers.13.block_sparse_moe.experts.2.w2.weight": "model-00004-of-00007.safetensors", + "model.layers.13.block_sparse_moe.experts.3.w1.weight": "model-00004-of-00007.safetensors", + "model.layers.13.block_sparse_moe.experts.3.w3.weight": "model-00004-of-00007.safetensors", + "model.layers.13.block_sparse_moe.experts.3.w2.weight": "model-00004-of-00007.safetensors", + "model.layers.13.block_sparse_moe.experts.4.w1.weight": "model-00004-of-00007.safetensors", + "model.layers.13.block_sparse_moe.experts.4.w3.weight": "model-00004-of-00007.safetensors", + "model.layers.13.block_sparse_moe.experts.4.w2.weight": "model-00004-of-00007.safetensors", + "model.layers.13.block_sparse_moe.experts.5.w1.weight": "model-00004-of-00007.safetensors", + "model.layers.13.block_sparse_moe.experts.5.w3.weight": "model-00004-of-00007.safetensors", + "model.layers.13.block_sparse_moe.experts.5.w2.weight": "model-00004-of-00007.safetensors", + "model.layers.13.block_sparse_moe.experts.6.w1.weight": "model-00004-of-00007.safetensors", + "model.layers.13.block_sparse_moe.experts.6.w3.weight": "model-00004-of-00007.safetensors", + "model.layers.13.block_sparse_moe.experts.6.w2.weight": "model-00004-of-00007.safetensors", + "model.layers.13.block_sparse_moe.experts.7.w1.weight": "model-00004-of-00007.safetensors", + "model.layers.13.block_sparse_moe.experts.7.w3.weight": "model-00004-of-00007.safetensors", + "model.layers.13.block_sparse_moe.experts.7.w2.weight": "model-00004-of-00007.safetensors", + "model.layers.14.input_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.14.post_attention_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.14.self_attn.q_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.14.self_attn.k_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.14.self_attn.v_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.14.self_attn.o_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.14.block_sparse_moe.gate.weight": "model-00004-of-00007.safetensors", + "model.layers.14.block_sparse_moe.experts.0.w1.weight": "model-00004-of-00007.safetensors", + "model.layers.14.block_sparse_moe.experts.0.w3.weight": "model-00004-of-00007.safetensors", + "model.layers.14.block_sparse_moe.experts.0.w2.weight": "model-00004-of-00007.safetensors", + "model.layers.14.block_sparse_moe.experts.1.w1.weight": "model-00004-of-00007.safetensors", + "model.layers.14.block_sparse_moe.experts.1.w3.weight": "model-00004-of-00007.safetensors", + "model.layers.14.block_sparse_moe.experts.1.w2.weight": "model-00004-of-00007.safetensors", + "model.layers.14.block_sparse_moe.experts.2.w1.weight": "model-00004-of-00007.safetensors", + "model.layers.14.block_sparse_moe.experts.2.w3.weight": "model-00004-of-00007.safetensors", + "model.layers.14.block_sparse_moe.experts.2.w2.weight": "model-00004-of-00007.safetensors", + "model.layers.14.block_sparse_moe.experts.3.w1.weight": "model-00004-of-00007.safetensors", + "model.layers.14.block_sparse_moe.experts.3.w3.weight": "model-00004-of-00007.safetensors", + "model.layers.14.block_sparse_moe.experts.3.w2.weight": "model-00004-of-00007.safetensors", + "model.layers.14.block_sparse_moe.experts.4.w1.weight": "model-00004-of-00007.safetensors", + "model.layers.14.block_sparse_moe.experts.4.w3.weight": "model-00004-of-00007.safetensors", + "model.layers.14.block_sparse_moe.experts.4.w2.weight": "model-00004-of-00007.safetensors", + "model.layers.14.block_sparse_moe.experts.5.w1.weight": "model-00004-of-00007.safetensors", + "model.layers.14.block_sparse_moe.experts.5.w3.weight": "model-00004-of-00007.safetensors", + "model.layers.14.block_sparse_moe.experts.5.w2.weight": "model-00004-of-00007.safetensors", + "model.layers.14.block_sparse_moe.experts.6.w1.weight": "model-00004-of-00007.safetensors", + "model.layers.14.block_sparse_moe.experts.6.w3.weight": "model-00004-of-00007.safetensors", + "model.layers.14.block_sparse_moe.experts.6.w2.weight": "model-00004-of-00007.safetensors", + "model.layers.14.block_sparse_moe.experts.7.w1.weight": "model-00004-of-00007.safetensors", + "model.layers.14.block_sparse_moe.experts.7.w3.weight": "model-00004-of-00007.safetensors", + "model.layers.14.block_sparse_moe.experts.7.w2.weight": "model-00004-of-00007.safetensors", + "model.layers.15.input_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.15.post_attention_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.15.self_attn.q_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.15.self_attn.k_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.15.self_attn.v_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.15.self_attn.o_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.15.block_sparse_moe.gate.weight": "model-00004-of-00007.safetensors", + "model.layers.15.block_sparse_moe.experts.0.w1.weight": "model-00004-of-00007.safetensors", + "model.layers.15.block_sparse_moe.experts.0.w3.weight": "model-00004-of-00007.safetensors", + "model.layers.15.block_sparse_moe.experts.0.w2.weight": "model-00004-of-00007.safetensors", + "model.layers.15.block_sparse_moe.experts.1.w1.weight": "model-00004-of-00007.safetensors", + "model.layers.15.block_sparse_moe.experts.1.w3.weight": "model-00004-of-00007.safetensors", + "model.layers.15.block_sparse_moe.experts.1.w2.weight": "model-00005-of-00007.safetensors", + "model.layers.15.block_sparse_moe.experts.2.w1.weight": "model-00005-of-00007.safetensors", + "model.layers.15.block_sparse_moe.experts.2.w3.weight": "model-00005-of-00007.safetensors", + "model.layers.15.block_sparse_moe.experts.2.w2.weight": "model-00005-of-00007.safetensors", + "model.layers.15.block_sparse_moe.experts.3.w1.weight": "model-00005-of-00007.safetensors", + "model.layers.15.block_sparse_moe.experts.3.w3.weight": "model-00005-of-00007.safetensors", + "model.layers.15.block_sparse_moe.experts.3.w2.weight": "model-00005-of-00007.safetensors", + "model.layers.15.block_sparse_moe.experts.4.w1.weight": "model-00005-of-00007.safetensors", + "model.layers.15.block_sparse_moe.experts.4.w3.weight": "model-00005-of-00007.safetensors", + "model.layers.15.block_sparse_moe.experts.4.w2.weight": "model-00005-of-00007.safetensors", + "model.layers.15.block_sparse_moe.experts.5.w1.weight": "model-00005-of-00007.safetensors", + "model.layers.15.block_sparse_moe.experts.5.w3.weight": "model-00005-of-00007.safetensors", + "model.layers.15.block_sparse_moe.experts.5.w2.weight": "model-00005-of-00007.safetensors", + "model.layers.15.block_sparse_moe.experts.6.w1.weight": "model-00005-of-00007.safetensors", + "model.layers.15.block_sparse_moe.experts.6.w3.weight": "model-00005-of-00007.safetensors", + "model.layers.15.block_sparse_moe.experts.6.w2.weight": "model-00005-of-00007.safetensors", + "model.layers.15.block_sparse_moe.experts.7.w1.weight": "model-00005-of-00007.safetensors", + "model.layers.15.block_sparse_moe.experts.7.w3.weight": "model-00005-of-00007.safetensors", + "model.layers.15.block_sparse_moe.experts.7.w2.weight": "model-00005-of-00007.safetensors", + "model.layers.16.input_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.16.post_attention_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.16.self_attn.q_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.16.self_attn.k_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.16.self_attn.v_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.16.self_attn.o_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.16.block_sparse_moe.gate.weight": "model-00005-of-00007.safetensors", + "model.layers.16.block_sparse_moe.experts.0.w1.weight": "model-00005-of-00007.safetensors", + "model.layers.16.block_sparse_moe.experts.0.w3.weight": "model-00005-of-00007.safetensors", + "model.layers.16.block_sparse_moe.experts.0.w2.weight": "model-00005-of-00007.safetensors", + "model.layers.16.block_sparse_moe.experts.1.w1.weight": "model-00005-of-00007.safetensors", + "model.layers.16.block_sparse_moe.experts.1.w3.weight": "model-00005-of-00007.safetensors", + "model.layers.16.block_sparse_moe.experts.1.w2.weight": "model-00005-of-00007.safetensors", + "model.layers.16.block_sparse_moe.experts.2.w1.weight": "model-00005-of-00007.safetensors", + "model.layers.16.block_sparse_moe.experts.2.w3.weight": "model-00005-of-00007.safetensors", + "model.layers.16.block_sparse_moe.experts.2.w2.weight": "model-00005-of-00007.safetensors", + "model.layers.16.block_sparse_moe.experts.3.w1.weight": "model-00005-of-00007.safetensors", + "model.layers.16.block_sparse_moe.experts.3.w3.weight": "model-00005-of-00007.safetensors", + "model.layers.16.block_sparse_moe.experts.3.w2.weight": "model-00005-of-00007.safetensors", + "model.layers.16.block_sparse_moe.experts.4.w1.weight": "model-00005-of-00007.safetensors", + "model.layers.16.block_sparse_moe.experts.4.w3.weight": "model-00005-of-00007.safetensors", + "model.layers.16.block_sparse_moe.experts.4.w2.weight": "model-00005-of-00007.safetensors", + "model.layers.16.block_sparse_moe.experts.5.w1.weight": "model-00005-of-00007.safetensors", + "model.layers.16.block_sparse_moe.experts.5.w3.weight": "model-00005-of-00007.safetensors", + "model.layers.16.block_sparse_moe.experts.5.w2.weight": "model-00005-of-00007.safetensors", + "model.layers.16.block_sparse_moe.experts.6.w1.weight": "model-00005-of-00007.safetensors", + "model.layers.16.block_sparse_moe.experts.6.w3.weight": "model-00005-of-00007.safetensors", + "model.layers.16.block_sparse_moe.experts.6.w2.weight": "model-00005-of-00007.safetensors", + "model.layers.16.block_sparse_moe.experts.7.w1.weight": "model-00005-of-00007.safetensors", + "model.layers.16.block_sparse_moe.experts.7.w3.weight": "model-00005-of-00007.safetensors", + "model.layers.16.block_sparse_moe.experts.7.w2.weight": "model-00005-of-00007.safetensors", + "model.layers.17.input_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.17.post_attention_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.17.self_attn.q_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.17.self_attn.k_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.17.self_attn.v_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.17.self_attn.o_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.17.block_sparse_moe.gate.weight": "model-00005-of-00007.safetensors", + "model.layers.17.block_sparse_moe.experts.0.w1.weight": "model-00005-of-00007.safetensors", + "model.layers.17.block_sparse_moe.experts.0.w3.weight": "model-00005-of-00007.safetensors", + "model.layers.17.block_sparse_moe.experts.0.w2.weight": "model-00005-of-00007.safetensors", + "model.layers.17.block_sparse_moe.experts.1.w1.weight": "model-00005-of-00007.safetensors", + "model.layers.17.block_sparse_moe.experts.1.w3.weight": "model-00005-of-00007.safetensors", + "model.layers.17.block_sparse_moe.experts.1.w2.weight": "model-00005-of-00007.safetensors", + "model.layers.17.block_sparse_moe.experts.2.w1.weight": "model-00005-of-00007.safetensors", + "model.layers.17.block_sparse_moe.experts.2.w3.weight": "model-00005-of-00007.safetensors", + "model.layers.17.block_sparse_moe.experts.2.w2.weight": "model-00005-of-00007.safetensors", + "model.layers.17.block_sparse_moe.experts.3.w1.weight": "model-00005-of-00007.safetensors", + "model.layers.17.block_sparse_moe.experts.3.w3.weight": "model-00005-of-00007.safetensors", + "model.layers.17.block_sparse_moe.experts.3.w2.weight": "model-00005-of-00007.safetensors", + "model.layers.17.block_sparse_moe.experts.4.w1.weight": "model-00005-of-00007.safetensors", + "model.layers.17.block_sparse_moe.experts.4.w3.weight": "model-00005-of-00007.safetensors", + "model.layers.17.block_sparse_moe.experts.4.w2.weight": "model-00005-of-00007.safetensors", + "model.layers.17.block_sparse_moe.experts.5.w1.weight": "model-00005-of-00007.safetensors", + "model.layers.17.block_sparse_moe.experts.5.w3.weight": "model-00005-of-00007.safetensors", + "model.layers.17.block_sparse_moe.experts.5.w2.weight": "model-00005-of-00007.safetensors", + "model.layers.17.block_sparse_moe.experts.6.w1.weight": "model-00005-of-00007.safetensors", + "model.layers.17.block_sparse_moe.experts.6.w3.weight": "model-00005-of-00007.safetensors", + "model.layers.17.block_sparse_moe.experts.6.w2.weight": "model-00005-of-00007.safetensors", + "model.layers.17.block_sparse_moe.experts.7.w1.weight": "model-00005-of-00007.safetensors", + "model.layers.17.block_sparse_moe.experts.7.w3.weight": "model-00005-of-00007.safetensors", + "model.layers.17.block_sparse_moe.experts.7.w2.weight": "model-00005-of-00007.safetensors", + "model.layers.18.input_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.18.post_attention_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.18.self_attn.q_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.18.self_attn.k_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.18.self_attn.v_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.18.self_attn.o_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.18.block_sparse_moe.gate.weight": "model-00005-of-00007.safetensors", + "model.layers.18.block_sparse_moe.experts.0.w1.weight": "model-00005-of-00007.safetensors", + "model.layers.18.block_sparse_moe.experts.0.w3.weight": "model-00005-of-00007.safetensors", + "model.layers.18.block_sparse_moe.experts.0.w2.weight": "model-00005-of-00007.safetensors", + "model.layers.18.block_sparse_moe.experts.1.w1.weight": "model-00005-of-00007.safetensors", + "model.layers.18.block_sparse_moe.experts.1.w3.weight": "model-00005-of-00007.safetensors", + "model.layers.18.block_sparse_moe.experts.1.w2.weight": "model-00005-of-00007.safetensors", + "model.layers.18.block_sparse_moe.experts.2.w1.weight": "model-00005-of-00007.safetensors", + "model.layers.18.block_sparse_moe.experts.2.w3.weight": "model-00005-of-00007.safetensors", + "model.layers.18.block_sparse_moe.experts.2.w2.weight": "model-00005-of-00007.safetensors", + "model.layers.18.block_sparse_moe.experts.3.w1.weight": "model-00005-of-00007.safetensors", + "model.layers.18.block_sparse_moe.experts.3.w3.weight": "model-00005-of-00007.safetensors", + "model.layers.18.block_sparse_moe.experts.3.w2.weight": "model-00005-of-00007.safetensors", + "model.layers.18.block_sparse_moe.experts.4.w1.weight": "model-00005-of-00007.safetensors", + "model.layers.18.block_sparse_moe.experts.4.w3.weight": "model-00005-of-00007.safetensors", + "model.layers.18.block_sparse_moe.experts.4.w2.weight": "model-00005-of-00007.safetensors", + "model.layers.18.block_sparse_moe.experts.5.w1.weight": "model-00005-of-00007.safetensors", + "model.layers.18.block_sparse_moe.experts.5.w3.weight": "model-00005-of-00007.safetensors", + "model.layers.18.block_sparse_moe.experts.5.w2.weight": "model-00005-of-00007.safetensors", + "model.layers.18.block_sparse_moe.experts.6.w1.weight": "model-00005-of-00007.safetensors", + "model.layers.18.block_sparse_moe.experts.6.w3.weight": "model-00005-of-00007.safetensors", + "model.layers.18.block_sparse_moe.experts.6.w2.weight": "model-00005-of-00007.safetensors", + "model.layers.18.block_sparse_moe.experts.7.w1.weight": "model-00005-of-00007.safetensors", + "model.layers.18.block_sparse_moe.experts.7.w3.weight": "model-00005-of-00007.safetensors", + "model.layers.18.block_sparse_moe.experts.7.w2.weight": "model-00005-of-00007.safetensors", + "model.layers.19.input_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.19.post_attention_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.19.self_attn.q_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.19.self_attn.k_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.19.self_attn.v_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.19.self_attn.o_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.19.block_sparse_moe.gate.weight": "model-00005-of-00007.safetensors", + "model.layers.19.block_sparse_moe.experts.0.w1.weight": "model-00005-of-00007.safetensors", + "model.layers.19.block_sparse_moe.experts.0.w3.weight": "model-00005-of-00007.safetensors", + "model.layers.19.block_sparse_moe.experts.0.w2.weight": "model-00005-of-00007.safetensors", + "model.layers.19.block_sparse_moe.experts.1.w1.weight": "model-00005-of-00007.safetensors", + "model.layers.19.block_sparse_moe.experts.1.w3.weight": "model-00005-of-00007.safetensors", + "model.layers.19.block_sparse_moe.experts.1.w2.weight": "model-00005-of-00007.safetensors", + "model.layers.19.block_sparse_moe.experts.2.w1.weight": "model-00006-of-00007.safetensors", + "model.layers.19.block_sparse_moe.experts.2.w3.weight": "model-00006-of-00007.safetensors", + "model.layers.19.block_sparse_moe.experts.2.w2.weight": "model-00006-of-00007.safetensors", + "model.layers.19.block_sparse_moe.experts.3.w1.weight": "model-00006-of-00007.safetensors", + "model.layers.19.block_sparse_moe.experts.3.w3.weight": "model-00006-of-00007.safetensors", + "model.layers.19.block_sparse_moe.experts.3.w2.weight": "model-00006-of-00007.safetensors", + "model.layers.19.block_sparse_moe.experts.4.w1.weight": "model-00006-of-00007.safetensors", + "model.layers.19.block_sparse_moe.experts.4.w3.weight": "model-00006-of-00007.safetensors", + "model.layers.19.block_sparse_moe.experts.4.w2.weight": "model-00006-of-00007.safetensors", + "model.layers.19.block_sparse_moe.experts.5.w1.weight": "model-00006-of-00007.safetensors", + "model.layers.19.block_sparse_moe.experts.5.w3.weight": "model-00006-of-00007.safetensors", + "model.layers.19.block_sparse_moe.experts.5.w2.weight": "model-00006-of-00007.safetensors", + "model.layers.19.block_sparse_moe.experts.6.w1.weight": "model-00006-of-00007.safetensors", + "model.layers.19.block_sparse_moe.experts.6.w3.weight": "model-00006-of-00007.safetensors", + "model.layers.19.block_sparse_moe.experts.6.w2.weight": "model-00006-of-00007.safetensors", + "model.layers.19.block_sparse_moe.experts.7.w1.weight": "model-00006-of-00007.safetensors", + "model.layers.19.block_sparse_moe.experts.7.w3.weight": "model-00006-of-00007.safetensors", + "model.layers.19.block_sparse_moe.experts.7.w2.weight": "model-00006-of-00007.safetensors", + "model.layers.20.input_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.20.post_attention_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.20.self_attn.q_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.20.self_attn.k_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.20.self_attn.v_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.20.self_attn.o_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.20.block_sparse_moe.gate.weight": "model-00006-of-00007.safetensors", + "model.layers.20.block_sparse_moe.experts.0.w1.weight": "model-00006-of-00007.safetensors", + "model.layers.20.block_sparse_moe.experts.0.w3.weight": "model-00006-of-00007.safetensors", + "model.layers.20.block_sparse_moe.experts.0.w2.weight": "model-00006-of-00007.safetensors", + "model.layers.20.block_sparse_moe.experts.1.w1.weight": "model-00006-of-00007.safetensors", + "model.layers.20.block_sparse_moe.experts.1.w3.weight": "model-00006-of-00007.safetensors", + "model.layers.20.block_sparse_moe.experts.1.w2.weight": "model-00006-of-00007.safetensors", + "model.layers.20.block_sparse_moe.experts.2.w1.weight": "model-00006-of-00007.safetensors", + "model.layers.20.block_sparse_moe.experts.2.w3.weight": "model-00006-of-00007.safetensors", + "model.layers.20.block_sparse_moe.experts.2.w2.weight": "model-00006-of-00007.safetensors", + "model.layers.20.block_sparse_moe.experts.3.w1.weight": "model-00006-of-00007.safetensors", + "model.layers.20.block_sparse_moe.experts.3.w3.weight": "model-00006-of-00007.safetensors", + "model.layers.20.block_sparse_moe.experts.3.w2.weight": "model-00006-of-00007.safetensors", + "model.layers.20.block_sparse_moe.experts.4.w1.weight": "model-00006-of-00007.safetensors", + "model.layers.20.block_sparse_moe.experts.4.w3.weight": "model-00006-of-00007.safetensors", + "model.layers.20.block_sparse_moe.experts.4.w2.weight": "model-00006-of-00007.safetensors", + "model.layers.20.block_sparse_moe.experts.5.w1.weight": "model-00006-of-00007.safetensors", + "model.layers.20.block_sparse_moe.experts.5.w3.weight": "model-00006-of-00007.safetensors", + "model.layers.20.block_sparse_moe.experts.5.w2.weight": "model-00006-of-00007.safetensors", + "model.layers.20.block_sparse_moe.experts.6.w1.weight": "model-00006-of-00007.safetensors", + "model.layers.20.block_sparse_moe.experts.6.w3.weight": "model-00006-of-00007.safetensors", + "model.layers.20.block_sparse_moe.experts.6.w2.weight": "model-00006-of-00007.safetensors", + "model.layers.20.block_sparse_moe.experts.7.w1.weight": "model-00006-of-00007.safetensors", + "model.layers.20.block_sparse_moe.experts.7.w3.weight": "model-00006-of-00007.safetensors", + "model.layers.20.block_sparse_moe.experts.7.w2.weight": "model-00006-of-00007.safetensors", + "model.layers.21.input_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.21.post_attention_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.21.self_attn.q_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.21.self_attn.k_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.21.self_attn.v_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.21.self_attn.o_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.21.block_sparse_moe.gate.weight": "model-00006-of-00007.safetensors", + "model.layers.21.block_sparse_moe.experts.0.w1.weight": "model-00006-of-00007.safetensors", + "model.layers.21.block_sparse_moe.experts.0.w3.weight": "model-00006-of-00007.safetensors", + "model.layers.21.block_sparse_moe.experts.0.w2.weight": "model-00006-of-00007.safetensors", + "model.layers.21.block_sparse_moe.experts.1.w1.weight": "model-00006-of-00007.safetensors", + "model.layers.21.block_sparse_moe.experts.1.w3.weight": "model-00006-of-00007.safetensors", + "model.layers.21.block_sparse_moe.experts.1.w2.weight": "model-00006-of-00007.safetensors", + "model.layers.21.block_sparse_moe.experts.2.w1.weight": "model-00006-of-00007.safetensors", + "model.layers.21.block_sparse_moe.experts.2.w3.weight": "model-00006-of-00007.safetensors", + "model.layers.21.block_sparse_moe.experts.2.w2.weight": "model-00006-of-00007.safetensors", + "model.layers.21.block_sparse_moe.experts.3.w1.weight": "model-00006-of-00007.safetensors", + "model.layers.21.block_sparse_moe.experts.3.w3.weight": "model-00006-of-00007.safetensors", + "model.layers.21.block_sparse_moe.experts.3.w2.weight": "model-00006-of-00007.safetensors", + "model.layers.21.block_sparse_moe.experts.4.w1.weight": "model-00006-of-00007.safetensors", + "model.layers.21.block_sparse_moe.experts.4.w3.weight": "model-00006-of-00007.safetensors", + "model.layers.21.block_sparse_moe.experts.4.w2.weight": "model-00006-of-00007.safetensors", + "model.layers.21.block_sparse_moe.experts.5.w1.weight": "model-00006-of-00007.safetensors", + "model.layers.21.block_sparse_moe.experts.5.w3.weight": "model-00006-of-00007.safetensors", + "model.layers.21.block_sparse_moe.experts.5.w2.weight": "model-00006-of-00007.safetensors", + "model.layers.21.block_sparse_moe.experts.6.w1.weight": "model-00006-of-00007.safetensors", + "model.layers.21.block_sparse_moe.experts.6.w3.weight": "model-00006-of-00007.safetensors", + "model.layers.21.block_sparse_moe.experts.6.w2.weight": "model-00006-of-00007.safetensors", + "model.layers.21.block_sparse_moe.experts.7.w1.weight": "model-00006-of-00007.safetensors", + "model.layers.21.block_sparse_moe.experts.7.w3.weight": "model-00006-of-00007.safetensors", + "model.layers.21.block_sparse_moe.experts.7.w2.weight": "model-00006-of-00007.safetensors", + "model.layers.22.input_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.22.post_attention_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.22.self_attn.q_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.22.self_attn.k_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.22.self_attn.v_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.22.self_attn.o_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.22.block_sparse_moe.gate.weight": "model-00006-of-00007.safetensors", + "model.layers.22.block_sparse_moe.experts.0.w1.weight": "model-00006-of-00007.safetensors", + "model.layers.22.block_sparse_moe.experts.0.w3.weight": "model-00006-of-00007.safetensors", + "model.layers.22.block_sparse_moe.experts.0.w2.weight": "model-00006-of-00007.safetensors", + "model.layers.22.block_sparse_moe.experts.1.w1.weight": "model-00006-of-00007.safetensors", + "model.layers.22.block_sparse_moe.experts.1.w3.weight": "model-00006-of-00007.safetensors", + "model.layers.22.block_sparse_moe.experts.1.w2.weight": "model-00006-of-00007.safetensors", + "model.layers.22.block_sparse_moe.experts.2.w1.weight": "model-00006-of-00007.safetensors", + "model.layers.22.block_sparse_moe.experts.2.w3.weight": "model-00006-of-00007.safetensors", + "model.layers.22.block_sparse_moe.experts.2.w2.weight": "model-00006-of-00007.safetensors", + "model.layers.22.block_sparse_moe.experts.3.w1.weight": "model-00006-of-00007.safetensors", + "model.layers.22.block_sparse_moe.experts.3.w3.weight": "model-00006-of-00007.safetensors", + "model.layers.22.block_sparse_moe.experts.3.w2.weight": "model-00006-of-00007.safetensors", + "model.layers.22.block_sparse_moe.experts.4.w1.weight": "model-00006-of-00007.safetensors", + "model.layers.22.block_sparse_moe.experts.4.w3.weight": "model-00006-of-00007.safetensors", + "model.layers.22.block_sparse_moe.experts.4.w2.weight": "model-00006-of-00007.safetensors", + "model.layers.22.block_sparse_moe.experts.5.w1.weight": "model-00006-of-00007.safetensors", + "model.layers.22.block_sparse_moe.experts.5.w3.weight": "model-00006-of-00007.safetensors", + "model.layers.22.block_sparse_moe.experts.5.w2.weight": "model-00006-of-00007.safetensors", + "model.layers.22.block_sparse_moe.experts.6.w1.weight": "model-00006-of-00007.safetensors", + "model.layers.22.block_sparse_moe.experts.6.w3.weight": "model-00006-of-00007.safetensors", + "model.layers.22.block_sparse_moe.experts.6.w2.weight": "model-00006-of-00007.safetensors", + "model.layers.22.block_sparse_moe.experts.7.w1.weight": "model-00006-of-00007.safetensors", + "model.layers.22.block_sparse_moe.experts.7.w3.weight": "model-00006-of-00007.safetensors", + "model.layers.22.block_sparse_moe.experts.7.w2.weight": "model-00006-of-00007.safetensors", + "model.layers.23.input_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.23.post_attention_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.23.self_attn.q_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.23.self_attn.k_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.23.self_attn.v_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.23.self_attn.o_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.23.block_sparse_moe.gate.weight": "model-00006-of-00007.safetensors", + "model.layers.23.block_sparse_moe.experts.0.w1.weight": "model-00006-of-00007.safetensors", + "model.layers.23.block_sparse_moe.experts.0.w3.weight": "model-00006-of-00007.safetensors", + "model.layers.23.block_sparse_moe.experts.0.w2.weight": "model-00006-of-00007.safetensors", + "model.layers.23.block_sparse_moe.experts.1.w1.weight": "model-00006-of-00007.safetensors", + "model.layers.23.block_sparse_moe.experts.1.w3.weight": "model-00006-of-00007.safetensors", + "model.layers.23.block_sparse_moe.experts.1.w2.weight": "model-00006-of-00007.safetensors", + "model.layers.23.block_sparse_moe.experts.2.w1.weight": "model-00006-of-00007.safetensors", + "model.layers.23.block_sparse_moe.experts.2.w3.weight": "model-00007-of-00007.safetensors", + "model.layers.23.block_sparse_moe.experts.2.w2.weight": "model-00007-of-00007.safetensors", + "model.layers.23.block_sparse_moe.experts.3.w1.weight": "model-00007-of-00007.safetensors", + "model.layers.23.block_sparse_moe.experts.3.w3.weight": "model-00007-of-00007.safetensors", + "model.layers.23.block_sparse_moe.experts.3.w2.weight": "model-00007-of-00007.safetensors", + "model.layers.23.block_sparse_moe.experts.4.w1.weight": "model-00007-of-00007.safetensors", + "model.layers.23.block_sparse_moe.experts.4.w3.weight": "model-00007-of-00007.safetensors", + "model.layers.23.block_sparse_moe.experts.4.w2.weight": "model-00007-of-00007.safetensors", + "model.layers.23.block_sparse_moe.experts.5.w1.weight": "model-00007-of-00007.safetensors", + "model.layers.23.block_sparse_moe.experts.5.w3.weight": "model-00007-of-00007.safetensors", + "model.layers.23.block_sparse_moe.experts.5.w2.weight": "model-00007-of-00007.safetensors", + "model.layers.23.block_sparse_moe.experts.6.w1.weight": "model-00007-of-00007.safetensors", + "model.layers.23.block_sparse_moe.experts.6.w3.weight": "model-00007-of-00007.safetensors", + "model.layers.23.block_sparse_moe.experts.6.w2.weight": "model-00007-of-00007.safetensors", + "model.layers.23.block_sparse_moe.experts.7.w1.weight": "model-00007-of-00007.safetensors", + "model.layers.23.block_sparse_moe.experts.7.w3.weight": "model-00007-of-00007.safetensors", + "model.layers.23.block_sparse_moe.experts.7.w2.weight": "model-00007-of-00007.safetensors", + "model.norm.weight": "model-00007-of-00007.safetensors", + "lm_head.weight": "model-00007-of-00007.safetensors" + } +} \ No newline at end of file diff --git a/modeling_keural.py b/modeling_keural.py new file mode 100644 index 0000000..6857824 --- /dev/null +++ b/modeling_keural.py @@ -0,0 +1,43 @@ +""" +KeuralMoECausalLM — Keural Mixture-of-Experts Causal Language Model. + +Bilingual Korean-English MoE LLM trained entirely from scratch. +Developed by MKD Corp AI Research, Republic of Korea. + +Architecture: + - 14.83B total parameters (~7.42B active per token) + - 24 layers, hidden=4096, GQA 32/8 heads + - 8 experts per layer, top-2 routing + - Sliding window attention (512, alternating layers) + - RoPE theta=500,000, context length=4096 + - Vocabulary: 131,074 tokens (131,072 SPM + <|im_start|> + <|im_end|>) + +Load with trust_remote_code=True: + from transformers import AutoModelForCausalLM, AutoTokenizer + model = AutoModelForCausalLM.from_pretrained( + "mkd-hossain/keural-sft3-final", + torch_dtype="bfloat16", + device_map="auto", + trust_remote_code=True, + ) +""" + +from transformers import MixtralForCausalLM +from transformers.utils import logging + +try: + from configuration_keural import KeuralConfig +except ImportError: + from .configuration_keural import KeuralConfig + +logger = logging.get_logger(__name__) + + +class KeuralMoECausalLM(MixtralForCausalLM): + """ + Keural MoE Causal Language Model. + Bilingual Korean-English 14.83B MoE LLM trained from scratch by MKD Corp AI Research. + """ + + config_class = KeuralConfig + _keys_to_ignore_on_load_missing = ["lm_head.weight"] diff --git a/special_tokens_map.json b/special_tokens_map.json new file mode 100644 index 0000000..7453a70 --- /dev/null +++ b/special_tokens_map.json @@ -0,0 +1,30 @@ +{ + "bos_token": { + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false + }, + "eos_token": { + "content": "<|im_end|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false + }, + "unk_token": { + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false + }, + "pad_token": { + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false + } +} \ No newline at end of file diff --git a/tokenizer.model b/tokenizer.model new file mode 100644 index 0000000..1b292cd --- /dev/null +++ b/tokenizer.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b982818ea2f2057ba791e2006d17683799f1d8ceb9c91322018a638c4ec4b170 +size 2657284 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..1dc5ce8 --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,16 @@ +{ + "bos_token": "", + "eos_token": "<|im_end|>", + "unk_token": "", + "pad_token": "", + "model_max_length": 4096, + "tokenizer_class": "LlamaTokenizer", + "sp_model_kwargs": {}, + "legacy": false, + "chat_template": "{% for message in messages %}<|im_start|>{{ message['role'] }}\n{{ message['content'] }}<|im_end|>\n{% endfor %}<|im_start|>assistant\n", + "added_tokens_decoder": { + "131072": {"content": "<|im_start|>", "special": true, "single_word": false, "lstrip": false, "rstrip": false, "normalized": false}, + "131073": {"content": "<|im_end|>", "special": true, "single_word": false, "lstrip": false, "rstrip": false, "normalized": false} + }, + "additional_special_tokens": ["<|im_start|>", "<|im_end|>"] +}