commit edbb844d7df09e0cf22864f6bea81eed2a2c5543 Author: ModelHub XC Date: Sun Sep 20 17:46:19 2026 +0800 初始化项目，由ModelHub XC社区提供模型 Model: ai-sage/GigaChat3.1-10B-A1.8B-bf16 Source: Original Platform diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..dd236be --- /dev/null +++ b/.gitattributes @@ -0,0 +1,55 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bin.* filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zstandard filter=lfs diff=lfs merge=lfs -text +*.tfevents* filter=lfs diff=lfs merge=lfs -text +*.db* filter=lfs diff=lfs merge=lfs -text +*.ark* filter=lfs diff=lfs merge=lfs -text +**/*ckpt*data* filter=lfs diff=lfs merge=lfs -text +**/*ckpt*.meta filter=lfs diff=lfs merge=lfs -text +**/*ckpt*.index filter=lfs diff=lfs merge=lfs -text + +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.gguf* filter=lfs diff=lfs merge=lfs -text +*.ggml filter=lfs diff=lfs merge=lfs -text +*.llamafile* filter=lfs diff=lfs merge=lfs -text +*.pt2 filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text + +model-00002-of-00005.safetensors filter=lfs diff=lfs merge=lfs -text +model-00000-of-00005.safetensors filter=lfs diff=lfs merge=lfs -text +model-00003-of-00005.safetensors filter=lfs diff=lfs merge=lfs -text +model-00004-of-00005.safetensors filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text +model-00005-of-00005.safetensors filter=lfs diff=lfs merge=lfs -text +model-00001-of-00005.safetensors filter=lfs diff=lfs merge=lfs -text \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..593b328 --- /dev/null +++ b/README.md @@ -0,0 +1,426 @@ +--- +license: mit +language: +- ru +- en +base_model: +- ai-sage/GigaChat3-10B-A1.8B-base +pipeline_tag: text-generation +library_name: transformers +tags: +- instruct +- moe +- multilingual +- fp8 +- tool-use +- long-context +--- + +# GigaChat 3.1 Lightning + +`GigaChat 3.1 Lightning` is the compact instruct model of the GigaChat 3.1 family. It is a Mixture-of-Experts (MoE) model with 10B total parameters and 1.8B active parameters, designed for fast multilingual assistant workloads, reasoning, code, function calling, and product-style deployment. + +This repository contains the BF16 version of the model. The FP8 checkpoint is available at [GigaChat3.1-10B-A1.8B](https://huggingface.co/ai-sage/GigaChat3.1-10B-A1.8B), and a GGUF version is available at [GigaChat3.1-10B-A1.8B-GGUF](https://huggingface.co/ai-sage/GigaChat3.1-10B-A1.8B-GGUF). + +More details can be found in the [Habr article](https://habr.com/en/companies/sberbank/articles/1014146/). + +## Model architecture + +GigaChat 3.1 Lightning uses a custom MoE architecture with the following key components. + +### Mixture-of-Experts (MoE) + +The model has 10B total parameters with 1.8B active parameters at inference time. This allows it to scale model capacity aggressively while keeping the active compute budget much lower than that of an equally large dense model. + +### Multi-head Latent Attention (MLA) + +Instead of standard multi-head attention, the model uses MLA, which compresses the KV cache into a latent representation. This reduces memory usage and improves inference throughput, especially in long-context settings. + +### Multi-Token Prediction (MTP) + +The model is trained with MTP, which allows it to predict multiple tokens per forward pass. In production systems, this can be used with speculative or parallel decoding techniques to improve throughput. + +## Training data + +The base GigaChat 3 training corpus spans 10 languages and includes books, academic material, code datasets, and mathematics datasets. All data goes through deduplication, language filtering, and automatic quality checks based on heuristics and classifiers. + +Synthetic data remains a major contributor to quality. Across the broader training corpus, we used approximately 5.5 trillion synthetic tokens, including: + +- question-answer data generated from source texts, +- reverse-prompt chains for structured data generation, +- model-authored notes embedded inside texts, +- millions of synthetic tasks with solutions in mathematics and olympiad-style programming, +- synthetic tests for code and reasoning tasks. + +For the 3.1 release, we made major data improvements: + +- Hard-domain expansion at Stage 1.5: stronger coverage of mathematics, finance, physics, engineering, biology, chemistry, and medicine. +- Stricter quality validation: our internal `Revisor` pipeline was extended with stronger checks for Markdown, LaTeX, and answer-format correctness. +- LLM-judge validation: SFT and DPO data is validated with judges selected for the task type and response structure. +- On-policy DPO data: preference pairs were generated from preview-model behavior, making them better aligned with real model failure modes. +- Better product-oriented data: we expanded data for search-and-citation scenarios, file-aware code interpretation, personalization, and agentic dialogues with executable tool calls. +- Improved answer style: we also revised formatting and writing guidelines to improve readability, correctness, and overall response quality. + + +## Post-training improvements + +### DPO in native FP8 + +Unlike the preview release, `GigaChat 3.1 Lightning` includes a full DPO stage. This stage was redesigned for the MoE setup and trained in native FP8, not just quantized after training. + +Important changes include: + +- MTP heads trained during DPO for better consistency between main-model predictions and MTP predictions, +- weighted gamma with exponential decay over long sequences, +- stronger tuning of batch size and DPO contribution, +- better robustness against loop-inducing failure modes. + +In our experiments, native FP8 DPO not only recovered the quality that could be lost with post-training FP8 quantization, but in some cases even exceeded the BF16 result while using substantially less memory. + +### Faster post-training + +We also optimized the SFT pipeline with a combination of sequence packing, dynamic sequence parallelism, and additional pipeline optimizations. This reduced training cost significantly and improved GPU utilization, especially on long-context workloads. + +## Inference + +One of the key advantages of `GigaChat3.1-10B-A1.8B` is its inference speed. The model (especially in MTP mode) demonstrates throughput comparable to that of significantly smaller dense models. +We measured this using vllm 0.17.1rc1.dev158+g600a039f5, concurrency=32, 1xH100 80gb SXM5. +[Link to code.](https://gist.github.com/chameleon-lizard/07c5fdc658da63b0fdf105ae5a752344) + +| Model | Output tps | Total tps | TPOT | Diff vs Lightning BF16 | +|---|---|---|---|---| +| GigaChat-3.1-Lightning BF16 | 2 866 | 5 832 | 9.52 | +0.0% | +| GigaChat-3.1-Lightning BF16 + MTP | 3 346 | 6 810 | 8.25 | +16.7% | +| GigaChat-3.1-Lightning FP8 | 3 382 | 6 883 | 7.63 | +18.0% | +| GigaChat-3.1-Lightning FP8 + MTP | 3 958 | 8 054 | 6.92 | +38.1% | +| YandexGPT-5-Lite-8B | 3 081 | 6 281 | 7.62 | +7.5% | + + +## Benchmark Results + +| Domain | Metric | GigaChat-3-Lightning | **GigaChat-3.1-Lightning** | Qwen3-1.7B-Instruct | Qwen3-4B-Instruct | SmolLM3 | gemma-3-4b-it | +|---|---|---:|---:|---:|---:|---:|---:| +| General | MMLU RU | 0.683 | 0.6803 | - | 0.597 | 0.500 | 0.519 | +| General | RUBQ | 0.652 | 0.6646 | - | 0.317 | 0.636 | 0.382 | +| General | MMLU PRO | 0.606 | 0.6176 | 0.410 | 0.685 | 0.501 | 0.410 | +| General | MMLU EN | 0.740 | 0.7298 | 0.600 | 0.708 | 0.599 | 0.594 | +| General | BBH | 0.453 | 0.5758 | 0.3317 | 0.717 | 0.416 | 0.131 | +| General | SuperGPQA | 0.273 | 0.2939 | 0.209 | 0.375 | 0.246 | 0.201 | +| Code | Human Eval Plus | 0.695 | 0.7317 | 0.628 | 0.878 | 0.701 | 0.713 | +| Total | Average | 0.586 | 0.631 | 0.458 | 0.612 | 0.514 | 0.421 | + + +## Arena Results + +| Arena | GigaChat-2-Lite-30.1 | GigaChat-3-Lightning | **GigaChat-3.1-Lightning** | YandexGPT-5-Lite-8B | SmolLM3 | gemma-3-4b-it | Qwen3-4B | Qwen3-4B-Instruct-2507 | +|---|---:|---:|---:|---:|---:|---:|---:|---:| +| Arena Hard Logs V3 | 23.700 | 14.3 | 46.700 | 17.9 | 18.1 | 38.7 | 27.7 | 61.5 | +| Validator SBS Pollux | 32.500 | 24.3 | 55.700 | 10.3 | 13.7 | 34.000 | 19.8 | 56.100 | +| Total Average | 28.100 | 19.3 | 51.200 | 14.1 | 15.9 | 36.35 | 23.75 | 58.800 | + + +## Usage Example + +### 1. `transformers` + +```python +import torch +from transformers import AutoTokenizer, AutoModelForCausalLM, GenerationConfig + +model_name = "ai-sage/GigaChat3.1-10B-A1.8B-bf16" +tokenizer = AutoTokenizer.from_pretrained(model_name) +model = AutoModelForCausalLM.from_pretrained( + model_name, + torch_dtype=torch.bfloat16, + device_map="auto", +) +model.generation_config = GenerationConfig.from_pretrained(model_name) +messages = [ + {"role": "user", "content": "Докажи теорему о неподвижной точке"} +] +prompt = tokenizer.apply_chat_template( + messages, + tokenize=False, + add_generation_prompt=True, +) +inputs = tokenizer(prompt, return_tensors="pt") +inputs = {k: v.to(model.device) for k, v in inputs.items()} +outputs = model.generate( + **inputs, + max_new_tokens=1000, +) +prompt_len = inputs["input_ids"].shape[1] +result = tokenizer.decode( + outputs[0][prompt_len:], + skip_special_tokens=True, +) +print(result) +``` + +### 2. `vLLM` + +Start the server +```shell +vllm serve ai-sage/GigaChat3.1-10B-A1.8B-bf16 \ + --dtype "auto" \ + --speculative-config '{"method": "mtp", "num_speculative_tokens": 1, "disable_padded_drafter_batch": false}' +``` + +Request example +```shell +curl http://localhost:8000/v1/chat/completions \ + -H "Content-Type: application/json" \ + -d '{ + "model": "ai-sage/GigaChat3.1-10B-A1.8B-bf16", + "messages": [ + { + "role": "user", + "content": "Докажи теорему о неподвижной точке" + } + ], + "max_tokens": 400, + "temperature": 0 + }' +``` + +### 3. `SGLang` + +Start the server +```shell +python -m sglang.launch_server \ + --model-path ai-sage/GigaChat3.1-10B-A1.8B-bf16 \ + --host 0.0.0.0 \ + --port 30000 \ + --dtype auto \ + --mem-fraction-static 0.88 \ + --speculative-algorithm EAGLE \ + --speculative-num-steps 1 \ + --speculative-eagle-topk 1 \ + --speculative-num-draft-tokens 2 +``` + +Request example +```shell +curl http://localhost:30000/v1/chat/completions \ + -H "Content-Type: application/json" \ + -d '{ + "model": "ai-sage/GigaChat3.1-10B-A1.8B-bf16", + "messages": [ + { + "role": "user", + "content": "Докажи теорему о неподвижной точке" + } + ], + "max_tokens": 1000, + "temperature": 0 + }' +``` + +## Function calling + +### 1. `transformers` + +
Click for a dropdown + +```python +import torch +import json +from transformers import AutoTokenizer, AutoModelForCausalLM, GenerationConfig + +FUNCTION_CALL_TOKEN = "<|function_call|>" + +def parse_function_and_content(completion_str: str): + completion_str = completion_str.strip() + + if FUNCTION_CALL_TOKEN not in completion_str: + return None, completion_str or None + + content_part, function_part = completion_str.split(FUNCTION_CALL_TOKEN, 1) + + content = content_part.strip() or None + function_part = function_part.strip() + + for suffix in ("", ""): + if function_part.endswith(suffix): + function_part = function_part[: -len(suffix)].strip() + + try: + function_call = json.loads(function_part) + except json.JSONDecodeError: + return None, content if content is not None else completion_str + + if not ( + isinstance(function_call, dict) + and "name" in function_call + and "arguments" in function_call + and isinstance(function_call["arguments"], dict) + ): + return None, content + + return function_call, content + + +model_name = "ai-sage/GigaChat3.1-10B-A1.8B-bf16" + +tokenizer = AutoTokenizer.from_pretrained(model_name) +model = AutoModelForCausalLM.from_pretrained( + model_name, + torch_dtype=torch.bfloat16, + device_map="auto", +) +model.generation_config = GenerationConfig.from_pretrained(model_name) + +tools = [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Получить информацию о текущей погоде в указанном городе.", + "parameters": { + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "Название города (например, Москва, Казань)." + } + }, + "required": ["city"] + } + } + } +] + +messages = [ + {"role": "user", "content": "Какая сейчас погода в Москве?"} +] + +prompt = tokenizer.apply_chat_template( + messages, + tools=tools, + tokenize=False, + add_generation_prompt=True, +) + +inputs = tokenizer(prompt, return_tensors="pt") +inputs = {k: v.to(model.device) for k, v in inputs.items()} + +with torch.inference_mode(): + outputs = model.generate( + **inputs, + max_new_tokens=1000, + ) + +prompt_len = inputs["input_ids"].shape[1] +completion = tokenizer.decode( + outputs[0][prompt_len:], + skip_special_tokens=False, +) + +function_call, content = parse_function_and_content(completion) +print(function_call, content) +``` + +
+ + +### 2. `vLLM` + +commit>=[293f036](https://github.com/vllm-project/vllm/tree/293f036e6d83ba05236d948e9800bc6d4d58a727) + +Start the server +```shell +vllm serve ai-sage/GigaChat3.1-10B-A1.8B-bf16 \ + --dtype "auto" \ + --speculative-config '{"method": "mtp", "num_speculative_tokens": 1, "disable_padded_drafter_batch": false}' \ + --enable-auto-tool-choice \ + --tool-call-parser gigachat3 +``` + +Request example +```shell +curl http://localhost:8000/v1/chat/completions \ + -H "Content-Type: application/json" \ + -d '{ + "model": "ai-sage/GigaChat3.1-10B-A1.8B-bf16", + "temperature": 0, + "messages": [ + { + "role": "user", + "content": "Какая сейчас погода в Москве?" + } + ], + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Получить информацию о текущей погоде в указанном городе.", + "parameters": { + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "Название города (например, Москва, Казань)." + } + }, + "required": ["city"] + } + } + } + ] +}' +``` + +### 3. `SGLang` + +commit>=[30a35ec](https://github.com/sgl-project/sglang/tree/30a35ecd90227216a72833aa29ef02dc067db827) + +Start the server +```shell +python -m sglang.launch_server \ + --model-path ai-sage/GigaChat3.1-10B-A1.8B-bf16 \ + --host 0.0.0.0 \ + --port 30000 \ + --dtype auto \ + --mem-fraction-static 0.88 \ + --speculative-algorithm EAGLE \ + --speculative-num-steps 1 \ + --speculative-eagle-topk 1 \ + --speculative-num-draft-tokens 2 + --tool-call-parser gigachat3 +``` + +Request example +```shell +curl http://localhost:30000/v1/chat/completions \ + -H "Content-Type: application/json" \ + -d '{ + "model": "ai-sage/GigaChat3.1-10B-A1.8B-bf16", + "temperature": 0, + "messages": [ + { + "role": "user", + "content": "Какая сейчас погода в Москве?" + } + ], + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Получить информацию о текущей погоде в указанном городе.", + "parameters": { + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "Название города (например, Москва, Казань)." + } + }, + "required": ["city"] + } + } + } + ] +}' +``` diff --git a/chat_template.jinja b/chat_template.jinja new file mode 100644 index 0000000..52d5ace --- /dev/null +++ b/chat_template.jinja @@ -0,0 +1,348 @@ +{#--------TOOL RENDERING FUNCTIONS---------#} + +{#--------------------------------------------------------------- + Converts JSON Schema (dict) to a TypeScript type definition +----------------------------------------------------------------#} +{%- macro json_schema_to_typescript(schema, indent="") -%} + {%- set ADDITIONAL_JSON_KEYS = ['format', 'maxItems', 'maximum', 'minItems', 'minimum', 'pattern'] -%} + {%- set ty = schema.get("type") -%} + + {# ---------------- OBJECT ---------------- #} + {%- if ty == "object" -%} + {{- "{\n" -}} + + {# Start building property list #} + {%- set props = schema.get("properties", {}) -%} + {%- set required = schema.get("required", []) -%} + {%- set has_additional_props = schema.get("additionalProperties") is defined -%} + {%- set additional_props_type = none -%} + {%- if has_additional_props -%} + {%- if schema.additionalProperties == true -%} + {%- set additional_props_type = {'type': 'any'} -%} + {%- elif schema.additionalProperties is mapping -%} + {%- set additional_props_type = schema.additionalProperties -%} + {%- endif -%} + {%- endif -%} + + {%- for key, val in props.items() -%} + {# ---------- Description Comments ---------- #} + {%- if "description" in val -%} + {%- for line in val['description'].split('\n') -%} + {%- if line.strip() -%} + {{- indent + '// ' + line + '\n' -}} + {%- endif -%} + {%- endfor -%} + {%- endif -%} + + {# ---------- Additional JSON Keys ---------- #} + {%- for add_key, add_val in val.items() -%} + {%- if add_key in ADDITIONAL_JSON_KEYS -%} + {%- if add_val is string -%} + {{- indent + '// ' + add_key + ': "' + add_val + '"' + '\n' -}} + {%- else -%} + {{- indent + '// ' + add_key + ': ' ~ add_val ~ '\n' -}} + {%- endif -%} + {%- endif -%} + {%- endfor -%} + + {# ---------- Property Definition ---------- #} + {%- set type_str = json_schema_to_typescript( + val, + indent + " " + ) -%} + + {{- indent + key + ('' if key in required else '?') + ': ' + type_str + ',' -}} + + {%- if "default" in val or "defalut_value" in val -%} + {%- set default = val.get("default", val.get("defalut_value")) -%} + {%- if default is string -%} + {{- ' // default: "' + default + '"' -}} + {%- else -%} + {{- ' // default: ' ~ default -}} + {%- endif -%} + {%- endif -%} + + {{- "\n" -}} + {%- endfor -%} + + {# Handle additionalProperties as index signature #} + {%- if has_additional_props and additional_props_type is not none -%} + {%- set additional_type_str = json_schema_to_typescript( + additional_props_type, + indent + " " + ) -%} + {{- indent + '[key: string]: ' + additional_type_str + '\n' -}} + {%- endif -%} + + {{- indent[: (indent|length - " "|length) ] + '}' -}} + + {# ---------------- STRING ---------------- #} + {%- elif ty == "string" -%} + {%- if schema.get("enum") -%} + {%- set ns = namespace(enum = []) -%} + {%- for en in schema['enum'] -%} + {%- set ns.enum = ns.enum + ['"' ~ en ~ '"'] -%} + {%- endfor -%} + {{- ns.enum | join(' | ') -}} + {%- elif schema.get("format", "none") in ['date-time', 'date'] -%} + {{- 'Date' -}} + {%- else -%} + {{- 'string' -}} + {%- endif -%} + + {# ---------------- NUMBER / INTEGER ---------------- #} + {%- elif ty in ["number", "integer"] -%} + {%- if schema.get("enum") -%} + {{- schema.enum | join(' | ') -}} + {%- else -%} + {{- 'number' -}} + {%- endif -%} + + {# ---------------- BOOLEAN ---------------- #} + {%- elif ty == "boolean" -%} + {{- 'boolean' -}} + + {# ---------------- ARRAY ---------------- #} + {%- elif ty == "array" -%} + {%- if "items" in schema -%} + {{- json_schema_to_typescript(schema['items'], indent) + '[]' -}} + {%- else -%} + {{- 'Array' -}} + {%- endif -%} + + {# ---------------- FALLBACK ---------------- #} + {%- else -%} + {{- 'any' -}} + {%- endif -%} +{%- endmacro -%} + +{#--------------------------------------------------------------- + Renders a namespace and its tool definitions in TypeScript style +----------------------------------------------------------------#} + +{%- macro render_tool_namespace(namespace_name, tools) -%} + {%- set ns = namespace(sections = ['namespace ' ~ namespace_name ~ ' {']) -%} + + {%- for tool in tools -%} + {%- if tool.function -%} + {%- set tool = tool.function -%} + {%- endif -%} + + {%- set ns_tool = namespace(content_lines=[]) -%} + + {# ---------- TOOL DESCRIPTION ---------- #} + {%- if tool.get('description') -%} + {%- for line in tool['description'].split('\n') -%} + {%- if line.strip() -%} + {%- set ns_tool.content_lines = ns_tool.content_lines + ['// ' ~ line] -%} + {%- endif -%} + {%- endfor -%} + {%- endif -%} + + {# ---------- TOOL SIGNATURE ---------- #} + {%- set main_body = "" -%} + {%- set params = tool.get("parameters") -%} + {%- if params and params.get("properties") -%} + {%- set param_type = json_schema_to_typescript(params, " ") -%} + {%- set main_body = 'type ' ~ tool.name ~ ' = (_: ' ~ param_type ~ ') => ' -%} + {%- else -%} + {%- set main_body = 'type ' ~ tool.name ~ ' = () => ' -%} + {%- endif -%} + + {# ---------- RETURN TYPE ---------- #} + {%- set return_params = tool.get("return_parameters") -%} + {%- if return_params and return_params.get("properties") -%} + {%- set return_type = json_schema_to_typescript(return_params, " ") -%} + {%- set main_body = main_body ~ return_type -%} + {%- else -%} + {%- set main_body = main_body ~ 'any' -%} + {%- endif -%} + + {%- set main_body = main_body ~ ';\n' -%} + + {%- set ns_tool.content_lines = ns_tool.content_lines + [main_body] -%} + + {# ---------- ADD TOOL TO SECTIONS ---------- #} + {%- set ns.sections = ns.sections + [ns_tool.content_lines | join('\n')] -%} + {%- endfor -%} + + {%- set ns.sections = ns.sections + ['} // namespace ' ~ namespace_name] -%} + + {{- ns.sections | join('\n') -}} +{%- endmacro -%} + + +{# ----------- MESSAGE RENDERING HELPER FUNCTIONS ------------ #} + +{%- macro render_function_call(call) -%} + {%- if call.function -%} + {%- set call = call.function -%} + {%- endif -%} + + {%- set arguments = call['arguments'] -%} + {%- if arguments is not string -%} + {%- set arguments = arguments| tojson(ensure_ascii=False) -%} + {%- endif -%} + + {{- '{"name": "' ~ call['name'] ~ '", "arguments": ' ~ arguments ~ '}' -}} +{%- endmacro -%} + + +{%- macro render_role_message(message, role=None) -%} + {%- if not role -%} + {%- set role = message["role"] -%} + {%- endif -%} + + {%- set message_content = message['content'] or '' -%} + {%- if message_content is not string -%} + {%- set message_content = message_content | tojson(ensure_ascii=False) -%} + {%- endif -%} + + {{- role + add_tokens.role_sep + message_content -}} + + {%- if message.tool_calls is defined and message.tool_calls -%} + {{- add_tokens.function_call + render_function_call(message.tool_calls[0]) -}} + {%- endif -%} + + {{- add_tokens.message_sep -}} + +{%- endmacro -%} + + + +{# ----- SPECIAL TOKENS ----- #} + +{%- set add_tokens = namespace( + role_sep="<|role_sep|>\n", + message_sep="<|message_sep|>\n\n", + function_call="<|function_call|>" +) -%} + +{# ----- DEFAULT DEVSYSTEM ----- #} + +{%- set DEVSYSTEM -%} + +Описание доступных в диалоге ролей. + +`developer system` +Сообщение, добавленное Сбером до основного диалога. Имеет самый высокий приоритет и определяет глобальные, неотменяемые условия (например, правила ведения диалога, политику безопасности, общий стиль ответов ассистента и пр.). + +`system` +Системная инструкция, добавляемая разработчиками или пользователем, но с приоритетом ниже, чем `developer system`. Обычно описывает инструкции ассистента, конкретный стиль ответа и другие условия для данного конкретного диалога. + +`user` +Сообщение или запрос от пользователя. Ассистент следует ему, если это не противоречит инструкциям более высокого приоритета (см. ). + +`user memory` +Последовательность наиболее актуальных долговременных фактов о пользователе на момент его запроса, представленная в виде JSON‑списка строк. Факты в ней перечислены в хронологическом порядке, то есть более новые факты дописываются в конец последовательности. При этом при изменении или удалении фактов записи о предыдущих фактах остаются в последовательности. Ассистент сохраняет факты с помощью функции и использует их в соответствии с указаниями из блока ниже. + +`added files` +Метаинформация о файлах, доступных для использования в диалоге, представленная в формате JSON. Содержит следующие ключи: id (уникальный идентификатор файла), name (имя файла), type (тип файла). + +`assistant` +Ответ ассистента на запрос пользователя. Если системная инструкция или пользователь не задаёт дополнительных правил для `assistant`, то такая реплика должна соответствовать указаниям из блока ниже. Список доступных для вызова функций содержится в последней реплике роли `available functions`. Название необходимой для вызова функции и аргументы будут сгенерированы после специального токена вызова функции. В своих репликах ассистент следует инструкциям в соответствии с . +Вызов функции осуществляется в строгом соответствии с инструкцией из блока . + +`function descriptions` +Описания функций в формате TypeScript. Функция — это специальный инструмент (или набор инструкций), который ассистент может вызвать для выполнения конкретных действий, вычислений или получения данных, необходимых для решения задачи пользователя. Каждое описание функции содержит блоки с именем, описанием, аргументами. Иногда описание содержит отдельные блоки с возвращаемыми параметрами и примерами применения, иллюстрирующими правильный вызов и аргументы. + +`available functions` +Список, который содержит названия функций, доступных для вызова. Если список не содержит элементов, то в следующем сообщении функции, доступные для вызова, отсутствуют. + +`function result` +Результат последнего вызова функции. + + + + +Ассистент умеет работать со следующими модальностями: текст, доступные функции. + + + + +В случае противоречия инструкций разных ролей в контексте диалога соблюдай приоритеты: +`developer system` > `system` > `user` > `function descriptions` > `function result` > `user memory` + + + + +Базовые инструкции для работы с функциями. + +Можно вызывать только те функции, которые доступны исходя из последнего сообщения `available functions`. + +Вызывай доступные функции в случае, если согласно их описанию такой вызов поможет дать более полный и/или точный ответ на запрос пользователя. Заполняй аргументы функций, используя информацию из контекста диалога. Если функция может помочь ответить на запрос, но для её обязательного аргумента отсутствует информация в контексте, уточни у пользователя недостающие данные перед вызовом функции. При недоступности необходимой функции или ошибке — кратко сообщи об этом пользователю и по возможности предложи альтернативу. + + + + +Правила использования фактов в долговременной памяти: + +Если в диалоге нет сообщения под ролью `user memory`, то это равносильно отсутствию долговременных фактов о пользователе в памяти. В таком случае информация о пользователе ограничена текущим диалогом, и новые факты не должны сохраняться. + + + + +GigaChat — нейросетевая модель искусственного интеллекта, созданная компанией Сбер в России. + +GigaChat старается отвечать на языке, на котором пользователь задал запрос. Если из запроса пользователя и контекста диалога язык определить невозможно, GigaChat использует русский. +GigaChat предоставляет подробные ответы на более сложные и открытые вопросы. +GigaChat в ответе не использует названия доступных функций. +GigaChat отвечает безопасно, в соответствии с действующим законодательством Российской Федерации, стараясь помочь пользователю решить задачу или поддержать беседу. + +Ты — GigaChat. + + + +Ниже будет приведён диалог. +В диалоге могут быть разнообразные роли, описанные в блоке . +Каждая реплика начинается с названия роли и специального токена, обозначающего конец полного наименования роли, а заканчивается специальным токеном конца реплики. +Твоя задача — продолжить диалог от последней указанной роли в соответствии с контекстом диалога. +{%- endset -%} + + +{#- ---------------------- RENDERING STARTS HERE ---------------------- -#} + + +{# ----- RENDER BOS TOKEN ----- #} +{{- bos_token -}} + + +{# ----- RENDER DEVSYSTEM ----- #} +{{- render_role_message({"role": "developer system", "content": DEVSYSTEM}) -}} + +{# ----- RENDER SYSTEM IF PRESENT ----- #} +{%- if messages and messages[0]['role'] == 'system' -%} + {{- render_role_message(messages[0]) -}} + {%- set messages = messages[1:] -%} +{%- else -%} + {{- render_role_message({"role": "system", "content": ""}) -}} +{%- endif -%} + +{# ----- RENDER TOOLS ----- #} +{%- if tools -%} + {%- set tools_content = ( + render_tool_namespace('functions', tools) + + "\n\n" + ) -%} + {{- render_role_message({'role': 'function descriptions', 'content': tools_content}) -}} +{%- endif -%} + +{# ----- MAIN MESSAGE LOOP ----- #} +{%- for message in messages -%} + + {# ----- TOOL MESSAGE -------#} + {%- if message['role'] == 'tool' -%} + {{- render_role_message(message, 'function result') -}} + + {# ----- OTHER MESSAGES ----- #} + {%- else -%} + {{- render_role_message(message) -}} + {%- endif -%} + + {# ----- ADDING GENERATION PROMPT ----- #} + + {%- if loop.last and add_generation_prompt and message['role'] != 'assistant' -%} + {{- 'assistant' + add_tokens.role_sep -}} + {%- endif -%} + +{%- endfor -%} \ No newline at end of file diff --git a/config.json b/config.json new file mode 100644 index 0000000..c5ecdd9 --- /dev/null +++ b/config.json @@ -0,0 +1,53 @@ +{ + "vocab_size": 128256, + "max_position_embeddings": 262144, + "hidden_size": 1536, + "intermediate_size": 8960, + "moe_intermediate_size": 1280, + "num_hidden_layers": 26, + "num_nextn_predict_layers": 1, + "num_attention_heads": 32, + "n_shared_experts": 1, + "n_routed_experts": 64, + "ep_size": 1, + "routed_scaling_factor": 1, + "kv_lora_rank": 512, + "q_lora_rank": null, + "qk_rope_head_dim": 64, + "v_head_dim": 192, + "qk_nope_head_dim": 128, + "topk_method": "noaux_tc", + "n_group": 1, + "topk_group": 1, + "num_experts_per_tok": 4, + "moe_layer_freq": 1, + "first_k_dense_replace": 1, + "norm_topk_prob": true, + "scoring_func": "sigmoid", + "num_key_value_heads": 32, + "hidden_act": "silu", + "initializer_range": 0.006, + "rms_norm_eps": 1e-06, + "use_cache": true, + "rope_theta": 100000, + "rope_scaling": { + "beta_fast": 32.0, + "beta_slow": 1.0, + "factor": 64.0, + "mscale": 1.0, + "mscale_all_dim": 1.0, + "original_max_position_embeddings": 4096, + "rope_type": "yarn" + }, + "attention_bias": false, + "attention_dropout": 0.0, + "tie_word_embeddings": false, + "architectures": [ + "DeepseekV3ForCausalLM" + ], + "bos_token_id": 1, + "eos_token_id": 2, + "transformers_version": "4.57.3", + "model_type": "deepseek_v3", + "dtype": "bfloat16" +} \ No newline at end of file diff --git a/configuration.json b/configuration.json new file mode 100644 index 0000000..bbeeda1 --- /dev/null +++ b/configuration.json @@ -0,0 +1 @@ +{"framework": "pytorch", "task": "text-generation", "allow_remote": true} \ No newline at end of file diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..37e924f --- /dev/null +++ b/generation_config.json @@ -0,0 +1,7 @@ +{ + "bos_token_id": 1, + "eos_token_id": 2, + "pad_token_id": 2, + "transformers_version": "4.57.3", + "_from_model_config": true +} \ No newline at end of file diff --git a/model-00000-of-00005.safetensors b/model-00000-of-00005.safetensors new file mode 100644 index 0000000..bbd0bf9 --- /dev/null +++ b/model-00000-of-00005.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:612f84d11aca544694ef674e58781d0c39ae4b2046f9cadeada79f1d37e581b6 +size 4986065392 diff --git a/model-00001-of-00005.safetensors b/model-00001-of-00005.safetensors new file mode 100644 index 0000000..2a90c1a --- /dev/null +++ b/model-00001-of-00005.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f104115bca75cec67bcfb162a71f57b4ab97de66e168d03adeb97a7d9e4e2cf +size 4085021224 diff --git a/model-00002-of-00005.safetensors b/model-00002-of-00005.safetensors new file mode 100644 index 0000000..f31b85f --- /dev/null +++ b/model-00002-of-00005.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9a3f9ffc60ec8e671687afb2260e7b6ab6b75267a4f50e39c494192edd15e1df +size 4085022040 diff --git a/model-00003-of-00005.safetensors b/model-00003-of-00005.safetensors new file mode 100644 index 0000000..b0cab39 --- /dev/null +++ b/model-00003-of-00005.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8fbb40d4301d9355248e37da641d03fa3633aad8c05b63dab3954d21a7b90071 +size 4085022240 diff --git a/model-00004-of-00005.safetensors b/model-00004-of-00005.safetensors new file mode 100644 index 0000000..3bf5b77 --- /dev/null +++ b/model-00004-of-00005.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:26c4261a7424c878e15e70c02d2429f1a16b39b74714068ac6e02aa2901879b0 +size 4085022240 diff --git a/model-00005-of-00005.safetensors b/model-00005-of-00005.safetensors new file mode 100644 index 0000000..5ec0dc2 --- /dev/null +++ b/model-00005-of-00005.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cf53c1a4db179dd8769499fdd6126b4cfb856fdcdc9026bf5edb2f328c026f8c +size 1634008560 diff --git a/model.safetensors.index.json b/model.safetensors.index.json new file mode 100644 index 0000000..fa2e4ea --- /dev/null +++ b/model.safetensors.index.json @@ -0,0 +1,5330 @@ +{ + "metadata": { + "total_size": 22959501568 + }, + "weight_map": { + "model.norm.weight": "model-00000-of-00005.safetensors", + "model.layers.26.self_attn.kv_a_layernorm.weight": "model-00000-of-00005.safetensors", + "model.layers.26.input_layernorm.weight": "model-00000-of-00005.safetensors", + "model.layers.26.post_attention_layernorm.weight": "model-00000-of-00005.safetensors", + "model.layers.26.enorm.weight": "model-00000-of-00005.safetensors", + "model.layers.26.hnorm.weight": "model-00000-of-00005.safetensors", + "model.layers.26.shared_head.norm.weight": "model-00000-of-00005.safetensors", + "lm_head.weight": "model-00000-of-00005.safetensors", + "model.layers.26.shared_head.head.weight": "model-00000-of-00005.safetensors", + "model.embed_tokens.weight": "model-00000-of-00005.safetensors", + "model.layers.26.embed_tokens.weight": "model-00000-of-00005.safetensors", + "model.layers.26.self_attn.q_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.self_attn.kv_b_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.self_attn.kv_a_proj_with_mqa.weight": "model-00000-of-00005.safetensors", + "model.layers.26.self_attn.o_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.gate.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.shared_experts.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.shared_experts.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.shared_experts.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.eh_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.gate.e_score_correction_bias": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.0.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.0.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.1.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.1.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.2.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.2.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.3.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.3.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.4.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.4.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.5.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.5.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.6.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.6.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.7.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.7.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.8.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.8.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.9.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.9.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.10.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.10.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.11.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.11.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.12.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.12.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.13.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.13.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.14.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.14.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.15.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.15.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.16.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.16.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.17.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.17.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.18.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.18.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.19.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.19.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.20.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.20.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.21.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.21.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.22.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.22.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.23.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.23.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.24.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.24.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.25.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.25.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.26.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.26.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.27.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.27.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.28.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.28.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.29.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.29.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.30.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.30.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.31.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.31.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.32.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.32.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.33.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.33.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.34.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.34.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.35.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.35.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.36.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.36.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.37.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.37.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.38.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.38.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.39.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.39.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.40.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.40.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.41.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.41.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.42.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.42.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.43.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.43.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.44.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.44.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.45.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.45.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.46.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.46.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.47.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.47.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.48.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.48.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.49.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.49.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.50.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.50.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.51.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.51.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.52.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.52.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.53.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.53.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.54.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.54.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.55.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.55.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.56.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.56.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.57.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.57.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.58.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.58.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.59.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.59.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.60.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.60.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.61.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.61.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.62.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.62.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.63.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.63.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.0.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.1.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.2.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.3.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.4.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.5.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.6.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.7.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.8.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.9.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.10.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.11.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.12.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.13.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.14.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.15.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.16.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.17.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.18.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.19.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.20.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.21.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.22.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.23.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.24.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.25.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.26.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.27.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.28.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.29.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.30.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.31.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.32.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.33.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.34.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.35.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.36.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.37.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.38.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.39.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.40.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.41.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.42.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.43.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.44.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.45.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.46.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.47.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.48.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.49.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.50.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.51.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.52.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.53.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.54.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.55.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.56.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.57.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.58.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.59.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.60.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.61.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.62.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.26.mlp.experts.63.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.0.self_attn.kv_a_layernorm.weight": "model-00000-of-00005.safetensors", + "model.layers.0.input_layernorm.weight": "model-00000-of-00005.safetensors", + "model.layers.0.post_attention_layernorm.weight": "model-00000-of-00005.safetensors", + "model.layers.0.self_attn.q_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.0.self_attn.kv_b_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.0.self_attn.kv_a_proj_with_mqa.weight": "model-00000-of-00005.safetensors", + "model.layers.0.self_attn.o_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.0.mlp.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.0.mlp.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.0.mlp.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.self_attn.kv_a_layernorm.weight": "model-00000-of-00005.safetensors", + "model.layers.1.input_layernorm.weight": "model-00000-of-00005.safetensors", + "model.layers.1.post_attention_layernorm.weight": "model-00000-of-00005.safetensors", + "model.layers.1.self_attn.q_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.self_attn.kv_b_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.self_attn.kv_a_proj_with_mqa.weight": "model-00000-of-00005.safetensors", + "model.layers.1.self_attn.o_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.gate.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.shared_experts.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.shared_experts.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.shared_experts.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.gate.e_score_correction_bias": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.0.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.0.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.1.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.1.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.2.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.2.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.3.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.3.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.4.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.4.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.5.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.5.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.6.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.6.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.7.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.7.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.8.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.8.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.9.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.9.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.10.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.10.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.11.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.11.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.12.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.12.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.13.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.13.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.14.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.14.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.15.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.15.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.16.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.16.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.17.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.17.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.18.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.18.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.19.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.19.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.20.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.20.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.21.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.21.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.22.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.22.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.23.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.23.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.24.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.24.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.25.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.25.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.26.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.26.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.27.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.27.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.28.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.28.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.29.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.29.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.30.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.30.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.31.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.31.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.32.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.32.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.33.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.33.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.34.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.34.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.35.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.35.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.36.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.36.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.37.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.37.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.38.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.38.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.39.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.39.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.40.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.40.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.41.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.41.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.42.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.42.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.43.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.43.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.44.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.44.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.45.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.45.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.46.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.46.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.47.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.47.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.48.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.48.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.49.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.49.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.50.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.50.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.51.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.51.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.52.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.52.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.53.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.53.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.54.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.54.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.55.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.55.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.56.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.56.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.57.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.57.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.58.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.58.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.59.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.59.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.60.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.60.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.61.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.61.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.62.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.62.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.63.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.63.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.0.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.1.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.2.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.3.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.4.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.5.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.6.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.7.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.8.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.9.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.10.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.11.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.12.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.13.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.14.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.15.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.16.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.17.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.18.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.19.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.20.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.21.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.22.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.23.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.24.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.25.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.26.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.27.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.28.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.29.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.30.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.31.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.32.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.33.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.34.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.35.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.36.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.37.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.38.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.39.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.40.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.41.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.42.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.43.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.44.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.45.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.46.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.47.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.48.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.49.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.50.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.51.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.52.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.53.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.54.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.55.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.56.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.57.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.58.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.59.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.60.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.61.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.62.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.1.mlp.experts.63.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.self_attn.kv_a_layernorm.weight": "model-00000-of-00005.safetensors", + "model.layers.2.input_layernorm.weight": "model-00000-of-00005.safetensors", + "model.layers.2.post_attention_layernorm.weight": "model-00000-of-00005.safetensors", + "model.layers.2.self_attn.q_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.self_attn.kv_b_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.self_attn.kv_a_proj_with_mqa.weight": "model-00000-of-00005.safetensors", + "model.layers.2.self_attn.o_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.gate.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.shared_experts.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.shared_experts.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.shared_experts.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.gate.e_score_correction_bias": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.0.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.0.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.1.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.1.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.2.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.2.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.3.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.3.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.4.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.4.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.5.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.5.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.6.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.6.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.7.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.7.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.8.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.8.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.9.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.9.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.10.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.10.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.11.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.11.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.12.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.12.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.13.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.13.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.14.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.14.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.15.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.15.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.16.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.16.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.17.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.17.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.18.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.18.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.19.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.19.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.20.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.20.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.21.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.21.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.22.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.22.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.23.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.23.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.24.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.24.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.25.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.25.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.26.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.26.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.27.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.27.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.28.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.28.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.29.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.29.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.30.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.30.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.31.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.31.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.32.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.32.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.33.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.33.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.34.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.34.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.35.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.35.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.36.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.36.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.37.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.37.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.38.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.38.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.39.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.39.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.40.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.40.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.41.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.41.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.42.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.42.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.43.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.43.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.44.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.44.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.45.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.45.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.46.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.46.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.47.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.47.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.48.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.48.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.49.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.49.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.50.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.50.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.51.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.51.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.52.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.52.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.53.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.53.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.54.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.54.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.55.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.55.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.56.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.56.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.57.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.57.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.58.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.58.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.59.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.59.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.60.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.60.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.61.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.61.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.62.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.62.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.63.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.63.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.0.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.1.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.2.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.3.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.4.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.5.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.6.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.7.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.8.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.9.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.10.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.11.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.12.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.13.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.14.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.15.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.16.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.17.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.18.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.19.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.20.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.21.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.22.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.23.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.24.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.25.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.26.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.27.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.28.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.29.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.30.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.31.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.32.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.33.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.34.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.35.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.36.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.37.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.38.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.39.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.40.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.41.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.42.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.43.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.44.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.45.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.46.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.47.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.48.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.49.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.50.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.51.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.52.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.53.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.54.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.55.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.56.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.57.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.58.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.59.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.60.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.61.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.62.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.2.mlp.experts.63.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.self_attn.kv_a_layernorm.weight": "model-00000-of-00005.safetensors", + "model.layers.3.input_layernorm.weight": "model-00000-of-00005.safetensors", + "model.layers.3.post_attention_layernorm.weight": "model-00000-of-00005.safetensors", + "model.layers.3.self_attn.q_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.self_attn.kv_b_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.self_attn.kv_a_proj_with_mqa.weight": "model-00000-of-00005.safetensors", + "model.layers.3.self_attn.o_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.gate.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.shared_experts.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.shared_experts.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.shared_experts.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.gate.e_score_correction_bias": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.0.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.0.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.1.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.1.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.2.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.2.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.3.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.3.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.4.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.4.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.5.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.5.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.6.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.6.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.7.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.7.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.8.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.8.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.9.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.9.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.10.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.10.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.11.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.11.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.12.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.12.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.13.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.13.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.14.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.14.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.15.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.15.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.16.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.16.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.17.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.17.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.18.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.18.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.19.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.19.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.20.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.20.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.21.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.21.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.22.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.22.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.23.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.23.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.24.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.24.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.25.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.25.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.26.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.26.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.27.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.27.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.28.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.28.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.29.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.29.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.30.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.30.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.31.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.31.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.32.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.32.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.33.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.33.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.34.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.34.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.35.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.35.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.36.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.36.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.37.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.37.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.38.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.38.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.39.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.39.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.40.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.40.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.41.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.41.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.42.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.42.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.43.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.43.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.44.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.44.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.45.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.45.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.46.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.46.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.47.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.47.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.48.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.48.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.49.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.49.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.50.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.50.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.51.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.51.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.52.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.52.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.53.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.53.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.54.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.54.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.55.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.55.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.56.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.56.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.57.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.57.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.58.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.58.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.59.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.59.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.60.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.60.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.61.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.61.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.62.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.62.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.63.gate_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.63.up_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.0.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.1.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.2.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.3.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.4.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.5.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.6.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.7.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.8.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.9.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.10.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.11.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.12.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.13.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.14.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.15.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.16.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.17.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.18.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.19.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.20.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.21.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.22.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.23.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.24.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.25.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.26.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.27.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.28.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.29.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.30.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.31.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.32.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.33.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.34.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.35.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.36.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.37.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.38.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.39.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.40.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.41.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.42.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.43.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.44.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.45.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.46.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.47.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.48.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.49.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.50.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.51.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.52.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.53.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.54.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.55.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.56.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.57.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.58.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.59.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.60.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.61.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.62.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.3.mlp.experts.63.down_proj.weight": "model-00000-of-00005.safetensors", + "model.layers.4.self_attn.kv_a_layernorm.weight": "model-00001-of-00005.safetensors", + "model.layers.4.input_layernorm.weight": "model-00001-of-00005.safetensors", + "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00005.safetensors", + "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.self_attn.kv_b_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.self_attn.kv_a_proj_with_mqa.weight": "model-00001-of-00005.safetensors", + "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.gate.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.shared_experts.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.shared_experts.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.shared_experts.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.gate.e_score_correction_bias": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.0.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.0.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.1.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.1.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.2.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.2.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.3.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.3.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.4.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.4.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.5.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.5.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.6.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.6.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.7.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.7.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.8.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.8.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.9.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.9.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.10.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.10.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.11.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.11.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.12.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.12.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.13.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.13.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.14.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.14.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.15.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.15.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.16.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.16.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.17.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.17.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.18.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.18.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.19.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.19.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.20.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.20.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.21.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.21.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.22.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.22.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.23.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.23.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.24.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.24.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.25.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.25.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.26.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.26.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.27.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.27.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.28.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.28.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.29.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.29.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.30.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.30.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.31.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.31.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.32.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.32.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.33.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.33.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.34.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.34.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.35.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.35.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.36.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.36.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.37.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.37.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.38.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.38.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.39.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.39.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.40.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.40.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.41.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.41.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.42.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.42.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.43.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.43.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.44.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.44.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.45.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.45.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.46.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.46.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.47.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.47.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.48.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.48.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.49.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.49.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.50.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.50.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.51.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.51.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.52.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.52.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.53.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.53.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.54.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.54.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.55.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.55.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.56.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.56.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.57.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.57.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.58.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.58.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.59.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.59.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.60.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.60.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.61.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.61.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.62.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.62.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.63.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.63.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.0.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.1.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.2.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.3.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.4.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.5.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.6.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.7.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.8.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.9.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.10.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.11.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.12.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.13.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.14.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.15.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.16.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.17.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.18.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.19.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.20.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.21.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.22.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.23.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.24.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.25.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.26.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.27.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.28.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.29.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.30.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.31.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.32.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.33.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.34.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.35.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.36.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.37.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.38.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.39.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.40.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.41.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.42.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.43.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.44.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.45.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.46.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.47.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.48.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.49.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.50.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.51.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.52.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.53.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.54.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.55.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.56.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.57.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.58.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.59.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.60.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.61.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.62.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.experts.63.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.self_attn.kv_a_layernorm.weight": "model-00001-of-00005.safetensors", + "model.layers.5.input_layernorm.weight": "model-00001-of-00005.safetensors", + "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00005.safetensors", + "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.self_attn.kv_b_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.self_attn.kv_a_proj_with_mqa.weight": "model-00001-of-00005.safetensors", + "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.gate.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.shared_experts.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.shared_experts.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.shared_experts.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.gate.e_score_correction_bias": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.0.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.0.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.1.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.1.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.2.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.2.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.3.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.3.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.4.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.4.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.5.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.5.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.6.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.6.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.7.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.7.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.8.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.8.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.9.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.9.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.10.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.10.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.11.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.11.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.12.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.12.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.13.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.13.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.14.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.14.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.15.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.15.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.16.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.16.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.17.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.17.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.18.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.18.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.19.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.19.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.20.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.20.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.21.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.21.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.22.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.22.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.23.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.23.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.24.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.24.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.25.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.25.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.26.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.26.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.27.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.27.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.28.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.28.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.29.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.29.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.30.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.30.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.31.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.31.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.32.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.32.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.33.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.33.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.34.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.34.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.35.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.35.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.36.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.36.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.37.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.37.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.38.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.38.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.39.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.39.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.40.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.40.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.41.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.41.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.42.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.42.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.43.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.43.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.44.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.44.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.45.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.45.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.46.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.46.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.47.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.47.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.48.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.48.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.49.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.49.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.50.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.50.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.51.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.51.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.52.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.52.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.53.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.53.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.54.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.54.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.55.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.55.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.56.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.56.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.57.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.57.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.58.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.58.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.59.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.59.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.60.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.60.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.61.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.61.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.62.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.62.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.63.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.63.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.0.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.1.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.2.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.3.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.4.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.5.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.6.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.7.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.8.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.9.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.10.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.11.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.12.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.13.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.14.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.15.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.16.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.17.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.18.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.19.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.20.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.21.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.22.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.23.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.24.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.25.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.26.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.27.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.28.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.29.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.30.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.31.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.32.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.33.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.34.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.35.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.36.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.37.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.38.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.39.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.40.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.41.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.42.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.43.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.44.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.45.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.46.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.47.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.48.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.49.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.50.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.51.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.52.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.53.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.54.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.55.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.56.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.57.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.58.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.59.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.60.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.61.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.62.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.experts.63.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.self_attn.kv_a_layernorm.weight": "model-00001-of-00005.safetensors", + "model.layers.6.input_layernorm.weight": "model-00001-of-00005.safetensors", + "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00005.safetensors", + "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.self_attn.kv_b_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.self_attn.kv_a_proj_with_mqa.weight": "model-00001-of-00005.safetensors", + "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.gate.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.shared_experts.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.shared_experts.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.shared_experts.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.gate.e_score_correction_bias": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.0.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.0.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.1.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.1.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.2.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.2.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.3.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.3.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.4.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.4.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.5.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.5.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.6.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.6.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.7.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.7.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.8.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.8.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.9.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.9.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.10.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.10.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.11.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.11.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.12.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.12.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.13.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.13.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.14.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.14.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.15.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.15.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.16.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.16.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.17.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.17.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.18.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.18.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.19.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.19.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.20.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.20.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.21.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.21.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.22.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.22.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.23.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.23.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.24.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.24.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.25.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.25.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.26.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.26.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.27.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.27.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.28.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.28.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.29.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.29.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.30.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.30.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.31.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.31.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.32.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.32.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.33.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.33.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.34.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.34.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.35.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.35.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.36.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.36.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.37.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.37.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.38.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.38.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.39.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.39.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.40.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.40.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.41.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.41.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.42.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.42.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.43.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.43.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.44.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.44.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.45.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.45.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.46.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.46.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.47.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.47.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.48.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.48.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.49.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.49.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.50.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.50.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.51.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.51.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.52.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.52.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.53.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.53.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.54.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.54.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.55.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.55.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.56.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.56.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.57.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.57.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.58.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.58.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.59.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.59.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.60.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.60.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.61.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.61.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.62.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.62.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.63.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.63.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.0.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.1.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.2.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.3.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.4.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.5.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.6.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.7.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.8.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.9.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.10.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.11.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.12.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.13.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.14.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.15.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.16.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.17.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.18.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.19.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.20.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.21.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.22.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.23.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.24.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.25.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.26.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.27.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.28.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.29.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.30.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.31.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.32.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.33.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.34.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.35.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.36.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.37.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.38.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.39.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.40.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.41.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.42.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.43.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.44.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.45.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.46.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.47.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.48.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.49.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.50.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.51.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.52.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.53.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.54.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.55.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.56.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.57.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.58.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.59.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.60.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.61.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.62.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.experts.63.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.self_attn.kv_a_layernorm.weight": "model-00001-of-00005.safetensors", + "model.layers.7.input_layernorm.weight": "model-00001-of-00005.safetensors", + "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00005.safetensors", + "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.self_attn.kv_b_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.self_attn.kv_a_proj_with_mqa.weight": "model-00001-of-00005.safetensors", + "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.gate.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.shared_experts.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.shared_experts.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.shared_experts.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.gate.e_score_correction_bias": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.0.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.0.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.1.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.1.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.2.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.2.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.3.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.3.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.4.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.4.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.5.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.5.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.6.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.6.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.7.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.7.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.8.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.8.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.9.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.9.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.10.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.10.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.11.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.11.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.12.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.12.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.13.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.13.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.14.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.14.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.15.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.15.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.16.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.16.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.17.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.17.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.18.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.18.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.19.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.19.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.20.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.20.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.21.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.21.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.22.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.22.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.23.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.23.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.24.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.24.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.25.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.25.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.26.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.26.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.27.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.27.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.28.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.28.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.29.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.29.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.30.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.30.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.31.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.31.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.32.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.32.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.33.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.33.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.34.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.34.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.35.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.35.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.36.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.36.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.37.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.37.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.38.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.38.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.39.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.39.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.40.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.40.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.41.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.41.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.42.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.42.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.43.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.43.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.44.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.44.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.45.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.45.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.46.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.46.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.47.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.47.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.48.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.48.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.49.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.49.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.50.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.50.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.51.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.51.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.52.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.52.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.53.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.53.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.54.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.54.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.55.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.55.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.56.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.56.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.57.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.57.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.58.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.58.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.59.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.59.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.60.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.60.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.61.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.61.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.62.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.62.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.63.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.63.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.0.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.1.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.2.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.3.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.4.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.5.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.6.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.7.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.8.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.9.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.10.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.11.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.12.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.13.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.14.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.15.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.16.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.17.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.18.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.19.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.20.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.21.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.22.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.23.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.24.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.25.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.26.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.27.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.28.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.29.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.30.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.31.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.32.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.33.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.34.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.35.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.36.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.37.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.38.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.39.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.40.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.41.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.42.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.43.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.44.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.45.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.46.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.47.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.48.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.49.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.50.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.51.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.52.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.53.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.54.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.55.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.56.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.57.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.58.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.59.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.60.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.61.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.62.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.experts.63.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.self_attn.kv_a_layernorm.weight": "model-00001-of-00005.safetensors", + "model.layers.8.input_layernorm.weight": "model-00001-of-00005.safetensors", + "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00005.safetensors", + "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.self_attn.kv_b_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.self_attn.kv_a_proj_with_mqa.weight": "model-00001-of-00005.safetensors", + "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.gate.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.shared_experts.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.shared_experts.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.shared_experts.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.gate.e_score_correction_bias": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.0.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.0.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.1.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.1.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.2.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.2.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.3.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.3.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.4.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.4.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.5.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.5.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.6.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.6.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.7.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.7.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.8.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.8.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.9.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.9.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.10.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.10.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.11.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.11.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.12.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.12.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.13.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.13.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.14.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.14.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.15.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.15.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.16.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.16.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.17.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.17.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.18.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.18.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.19.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.19.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.20.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.20.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.21.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.21.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.22.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.22.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.23.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.23.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.24.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.24.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.25.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.25.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.26.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.26.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.27.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.27.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.28.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.28.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.29.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.29.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.30.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.30.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.31.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.31.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.32.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.32.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.33.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.33.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.34.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.34.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.35.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.35.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.36.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.36.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.37.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.37.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.38.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.38.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.39.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.39.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.40.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.40.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.41.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.41.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.42.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.42.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.43.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.43.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.44.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.44.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.45.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.45.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.46.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.46.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.47.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.47.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.48.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.48.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.49.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.49.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.50.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.50.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.51.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.51.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.52.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.52.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.53.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.53.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.54.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.54.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.55.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.55.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.56.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.56.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.57.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.57.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.58.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.58.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.59.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.59.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.60.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.60.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.61.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.61.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.62.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.62.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.63.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.63.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.0.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.1.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.2.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.3.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.4.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.5.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.6.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.7.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.8.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.9.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.10.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.11.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.12.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.13.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.14.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.15.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.16.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.17.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.18.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.19.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.20.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.21.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.22.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.23.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.24.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.25.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.26.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.27.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.28.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.29.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.30.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.31.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.32.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.33.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.34.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.35.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.36.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.37.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.38.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.39.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.40.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.41.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.42.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.43.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.44.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.45.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.46.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.47.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.48.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.49.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.50.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.51.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.52.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.53.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.54.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.55.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.56.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.57.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.58.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.59.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.60.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.61.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.62.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.experts.63.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.9.self_attn.kv_a_layernorm.weight": "model-00002-of-00005.safetensors", + "model.layers.9.input_layernorm.weight": "model-00002-of-00005.safetensors", + "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00005.safetensors", + "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.self_attn.kv_b_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.self_attn.kv_a_proj_with_mqa.weight": "model-00002-of-00005.safetensors", + "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.gate.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.shared_experts.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.shared_experts.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.shared_experts.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.gate.e_score_correction_bias": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.0.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.0.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.1.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.1.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.2.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.2.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.3.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.3.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.4.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.4.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.5.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.5.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.6.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.6.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.7.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.7.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.8.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.8.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.9.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.9.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.10.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.10.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.11.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.11.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.12.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.12.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.13.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.13.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.14.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.14.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.15.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.15.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.16.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.16.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.17.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.17.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.18.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.18.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.19.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.19.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.20.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.20.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.21.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.21.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.22.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.22.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.23.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.23.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.24.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.24.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.25.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.25.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.26.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.26.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.27.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.27.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.28.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.28.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.29.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.29.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.30.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.30.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.31.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.31.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.32.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.32.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.33.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.33.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.34.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.34.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.35.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.35.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.36.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.36.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.37.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.37.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.38.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.38.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.39.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.39.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.40.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.40.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.41.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.41.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.42.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.42.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.43.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.43.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.44.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.44.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.45.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.45.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.46.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.46.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.47.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.47.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.48.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.48.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.49.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.49.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.50.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.50.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.51.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.51.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.52.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.52.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.53.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.53.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.54.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.54.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.55.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.55.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.56.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.56.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.57.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.57.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.58.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.58.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.59.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.59.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.60.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.60.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.61.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.61.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.62.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.62.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.63.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.63.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.0.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.1.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.2.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.3.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.4.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.5.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.6.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.7.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.8.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.9.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.10.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.11.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.12.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.13.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.14.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.15.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.16.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.17.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.18.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.19.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.20.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.21.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.22.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.23.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.24.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.25.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.26.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.27.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.28.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.29.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.30.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.31.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.32.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.33.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.34.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.35.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.36.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.37.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.38.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.39.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.40.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.41.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.42.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.43.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.44.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.45.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.46.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.47.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.48.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.49.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.50.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.51.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.52.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.53.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.54.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.55.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.56.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.57.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.58.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.59.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.60.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.61.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.62.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.experts.63.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.self_attn.kv_a_layernorm.weight": "model-00002-of-00005.safetensors", + "model.layers.10.input_layernorm.weight": "model-00002-of-00005.safetensors", + "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00005.safetensors", + "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.self_attn.kv_b_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.self_attn.kv_a_proj_with_mqa.weight": "model-00002-of-00005.safetensors", + "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.gate.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.shared_experts.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.shared_experts.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.shared_experts.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.gate.e_score_correction_bias": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.0.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.0.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.1.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.1.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.2.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.2.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.3.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.3.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.4.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.4.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.5.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.5.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.6.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.6.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.7.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.7.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.8.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.8.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.9.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.9.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.10.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.10.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.11.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.11.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.12.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.12.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.13.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.13.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.14.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.14.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.15.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.15.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.16.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.16.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.17.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.17.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.18.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.18.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.19.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.19.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.20.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.20.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.21.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.21.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.22.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.22.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.23.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.23.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.24.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.24.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.25.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.25.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.26.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.26.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.27.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.27.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.28.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.28.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.29.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.29.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.30.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.30.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.31.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.31.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.32.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.32.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.33.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.33.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.34.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.34.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.35.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.35.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.36.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.36.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.37.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.37.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.38.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.38.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.39.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.39.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.40.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.40.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.41.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.41.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.42.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.42.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.43.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.43.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.44.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.44.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.45.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.45.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.46.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.46.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.47.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.47.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.48.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.48.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.49.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.49.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.50.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.50.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.51.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.51.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.52.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.52.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.53.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.53.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.54.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.54.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.55.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.55.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.56.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.56.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.57.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.57.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.58.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.58.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.59.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.59.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.60.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.60.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.61.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.61.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.62.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.62.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.63.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.63.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.0.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.1.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.2.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.3.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.4.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.5.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.6.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.7.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.8.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.9.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.10.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.11.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.12.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.13.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.14.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.15.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.16.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.17.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.18.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.19.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.20.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.21.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.22.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.23.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.24.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.25.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.26.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.27.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.28.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.29.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.30.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.31.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.32.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.33.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.34.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.35.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.36.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.37.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.38.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.39.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.40.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.41.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.42.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.43.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.44.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.45.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.46.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.47.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.48.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.49.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.50.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.51.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.52.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.53.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.54.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.55.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.56.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.57.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.58.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.59.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.60.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.61.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.62.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.experts.63.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.self_attn.kv_a_layernorm.weight": "model-00002-of-00005.safetensors", + "model.layers.11.input_layernorm.weight": "model-00002-of-00005.safetensors", + "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00005.safetensors", + "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.self_attn.kv_b_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.self_attn.kv_a_proj_with_mqa.weight": "model-00002-of-00005.safetensors", + "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.gate.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.shared_experts.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.shared_experts.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.shared_experts.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.gate.e_score_correction_bias": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.0.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.0.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.1.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.1.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.2.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.2.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.3.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.3.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.4.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.4.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.5.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.5.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.6.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.6.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.7.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.7.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.8.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.8.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.9.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.9.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.10.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.10.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.11.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.11.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.12.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.12.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.13.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.13.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.14.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.14.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.15.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.15.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.16.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.16.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.17.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.17.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.18.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.18.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.19.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.19.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.20.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.20.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.21.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.21.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.22.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.22.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.23.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.23.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.24.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.24.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.25.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.25.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.26.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.26.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.27.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.27.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.28.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.28.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.29.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.29.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.30.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.30.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.31.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.31.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.32.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.32.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.33.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.33.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.34.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.34.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.35.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.35.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.36.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.36.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.37.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.37.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.38.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.38.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.39.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.39.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.40.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.40.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.41.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.41.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.42.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.42.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.43.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.43.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.44.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.44.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.45.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.45.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.46.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.46.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.47.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.47.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.48.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.48.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.49.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.49.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.50.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.50.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.51.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.51.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.52.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.52.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.53.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.53.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.54.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.54.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.55.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.55.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.56.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.56.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.57.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.57.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.58.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.58.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.59.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.59.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.60.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.60.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.61.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.61.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.62.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.62.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.63.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.63.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.0.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.1.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.2.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.3.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.4.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.5.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.6.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.7.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.8.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.9.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.10.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.11.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.12.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.13.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.14.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.15.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.16.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.17.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.18.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.19.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.20.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.21.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.22.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.23.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.24.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.25.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.26.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.27.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.28.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.29.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.30.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.31.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.32.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.33.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.34.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.35.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.36.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.37.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.38.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.39.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.40.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.41.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.42.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.43.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.44.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.45.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.46.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.47.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.48.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.49.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.50.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.51.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.52.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.53.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.54.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.55.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.56.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.57.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.58.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.59.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.60.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.61.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.62.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.experts.63.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.self_attn.kv_a_layernorm.weight": "model-00002-of-00005.safetensors", + "model.layers.12.input_layernorm.weight": "model-00002-of-00005.safetensors", + "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00005.safetensors", + "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.self_attn.kv_b_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.self_attn.kv_a_proj_with_mqa.weight": "model-00002-of-00005.safetensors", + "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.gate.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.shared_experts.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.shared_experts.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.shared_experts.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.gate.e_score_correction_bias": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.0.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.0.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.1.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.1.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.2.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.2.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.3.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.3.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.4.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.4.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.5.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.5.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.6.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.6.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.7.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.7.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.8.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.8.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.9.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.9.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.10.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.10.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.11.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.11.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.12.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.12.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.13.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.13.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.14.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.14.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.15.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.15.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.16.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.16.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.17.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.17.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.18.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.18.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.19.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.19.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.20.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.20.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.21.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.21.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.22.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.22.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.23.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.23.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.24.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.24.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.25.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.25.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.26.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.26.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.27.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.27.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.28.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.28.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.29.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.29.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.30.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.30.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.31.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.31.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.32.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.32.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.33.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.33.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.34.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.34.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.35.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.35.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.36.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.36.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.37.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.37.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.38.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.38.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.39.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.39.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.40.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.40.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.41.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.41.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.42.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.42.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.43.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.43.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.44.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.44.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.45.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.45.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.46.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.46.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.47.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.47.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.48.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.48.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.49.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.49.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.50.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.50.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.51.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.51.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.52.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.52.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.53.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.53.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.54.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.54.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.55.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.55.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.56.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.56.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.57.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.57.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.58.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.58.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.59.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.59.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.60.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.60.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.61.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.61.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.62.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.62.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.63.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.63.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.0.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.1.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.2.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.3.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.4.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.5.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.6.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.7.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.8.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.9.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.10.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.11.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.12.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.13.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.14.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.15.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.16.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.17.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.18.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.19.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.20.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.21.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.22.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.23.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.24.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.25.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.26.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.27.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.28.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.29.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.30.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.31.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.32.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.33.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.34.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.35.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.36.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.37.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.38.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.39.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.40.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.41.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.42.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.43.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.44.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.45.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.46.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.47.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.48.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.49.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.50.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.51.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.52.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.53.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.54.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.55.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.56.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.57.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.58.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.59.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.60.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.61.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.62.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.experts.63.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.self_attn.kv_a_layernorm.weight": "model-00002-of-00005.safetensors", + "model.layers.13.input_layernorm.weight": "model-00002-of-00005.safetensors", + "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00005.safetensors", + "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.self_attn.kv_b_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.self_attn.kv_a_proj_with_mqa.weight": "model-00002-of-00005.safetensors", + "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.gate.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.shared_experts.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.shared_experts.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.shared_experts.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.gate.e_score_correction_bias": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.0.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.0.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.1.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.1.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.2.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.2.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.3.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.3.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.4.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.4.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.5.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.5.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.6.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.6.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.7.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.7.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.8.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.8.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.9.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.9.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.10.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.10.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.11.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.11.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.12.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.12.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.13.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.13.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.14.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.14.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.15.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.15.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.16.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.16.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.17.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.17.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.18.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.18.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.19.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.19.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.20.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.20.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.21.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.21.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.22.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.22.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.23.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.23.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.24.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.24.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.25.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.25.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.26.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.26.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.27.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.27.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.28.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.28.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.29.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.29.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.30.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.30.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.31.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.31.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.32.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.32.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.33.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.33.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.34.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.34.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.35.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.35.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.36.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.36.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.37.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.37.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.38.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.38.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.39.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.39.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.40.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.40.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.41.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.41.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.42.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.42.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.43.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.43.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.44.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.44.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.45.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.45.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.46.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.46.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.47.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.47.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.48.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.48.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.49.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.49.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.50.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.50.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.51.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.51.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.52.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.52.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.53.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.53.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.54.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.54.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.55.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.55.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.56.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.56.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.57.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.57.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.58.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.58.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.59.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.59.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.60.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.60.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.61.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.61.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.62.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.62.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.63.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.63.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.0.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.1.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.2.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.3.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.4.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.5.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.6.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.7.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.8.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.9.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.10.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.11.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.12.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.13.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.14.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.15.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.16.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.17.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.18.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.19.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.20.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.21.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.22.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.23.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.24.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.25.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.26.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.27.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.28.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.29.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.30.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.31.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.32.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.33.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.34.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.35.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.36.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.37.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.38.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.39.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.40.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.41.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.42.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.43.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.44.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.45.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.46.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.47.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.48.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.49.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.50.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.51.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.52.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.53.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.54.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.55.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.56.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.57.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.58.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.59.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.60.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.61.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.62.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.experts.63.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.14.self_attn.kv_a_layernorm.weight": "model-00003-of-00005.safetensors", + "model.layers.14.input_layernorm.weight": "model-00003-of-00005.safetensors", + "model.layers.14.post_attention_layernorm.weight": "model-00003-of-00005.safetensors", + "model.layers.14.self_attn.q_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.self_attn.kv_b_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.self_attn.kv_a_proj_with_mqa.weight": "model-00003-of-00005.safetensors", + "model.layers.14.self_attn.o_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.gate.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.shared_experts.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.shared_experts.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.shared_experts.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.gate.e_score_correction_bias": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.0.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.0.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.1.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.1.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.2.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.2.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.3.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.3.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.4.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.4.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.5.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.5.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.6.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.6.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.7.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.7.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.8.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.8.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.9.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.9.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.10.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.10.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.11.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.11.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.12.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.12.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.13.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.13.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.14.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.14.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.15.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.15.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.16.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.16.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.17.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.17.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.18.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.18.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.19.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.19.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.20.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.20.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.21.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.21.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.22.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.22.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.23.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.23.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.24.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.24.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.25.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.25.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.26.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.26.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.27.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.27.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.28.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.28.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.29.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.29.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.30.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.30.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.31.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.31.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.32.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.32.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.33.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.33.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.34.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.34.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.35.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.35.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.36.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.36.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.37.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.37.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.38.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.38.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.39.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.39.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.40.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.40.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.41.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.41.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.42.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.42.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.43.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.43.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.44.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.44.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.45.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.45.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.46.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.46.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.47.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.47.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.48.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.48.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.49.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.49.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.50.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.50.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.51.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.51.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.52.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.52.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.53.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.53.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.54.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.54.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.55.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.55.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.56.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.56.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.57.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.57.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.58.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.58.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.59.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.59.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.60.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.60.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.61.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.61.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.62.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.62.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.63.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.63.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.0.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.1.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.2.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.3.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.4.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.5.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.6.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.7.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.8.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.9.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.10.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.11.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.12.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.13.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.14.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.15.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.16.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.17.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.18.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.19.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.20.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.21.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.22.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.23.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.24.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.25.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.26.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.27.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.28.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.29.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.30.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.31.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.32.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.33.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.34.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.35.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.36.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.37.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.38.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.39.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.40.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.41.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.42.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.43.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.44.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.45.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.46.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.47.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.48.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.49.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.50.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.51.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.52.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.53.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.54.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.55.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.56.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.57.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.58.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.59.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.60.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.61.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.62.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.14.mlp.experts.63.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.self_attn.kv_a_layernorm.weight": "model-00003-of-00005.safetensors", + "model.layers.15.input_layernorm.weight": "model-00003-of-00005.safetensors", + "model.layers.15.post_attention_layernorm.weight": "model-00003-of-00005.safetensors", + "model.layers.15.self_attn.q_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.self_attn.kv_b_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.self_attn.kv_a_proj_with_mqa.weight": "model-00003-of-00005.safetensors", + "model.layers.15.self_attn.o_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.gate.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.shared_experts.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.shared_experts.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.shared_experts.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.gate.e_score_correction_bias": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.0.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.0.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.1.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.1.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.2.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.2.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.3.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.3.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.4.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.4.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.5.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.5.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.6.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.6.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.7.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.7.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.8.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.8.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.9.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.9.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.10.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.10.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.11.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.11.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.12.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.12.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.13.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.13.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.14.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.14.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.15.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.15.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.16.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.16.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.17.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.17.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.18.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.18.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.19.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.19.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.20.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.20.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.21.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.21.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.22.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.22.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.23.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.23.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.24.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.24.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.25.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.25.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.26.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.26.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.27.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.27.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.28.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.28.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.29.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.29.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.30.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.30.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.31.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.31.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.32.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.32.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.33.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.33.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.34.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.34.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.35.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.35.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.36.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.36.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.37.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.37.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.38.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.38.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.39.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.39.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.40.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.40.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.41.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.41.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.42.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.42.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.43.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.43.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.44.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.44.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.45.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.45.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.46.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.46.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.47.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.47.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.48.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.48.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.49.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.49.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.50.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.50.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.51.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.51.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.52.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.52.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.53.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.53.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.54.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.54.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.55.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.55.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.56.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.56.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.57.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.57.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.58.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.58.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.59.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.59.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.60.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.60.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.61.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.61.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.62.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.62.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.63.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.63.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.0.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.1.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.2.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.3.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.4.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.5.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.6.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.7.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.8.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.9.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.10.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.11.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.12.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.13.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.14.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.15.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.16.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.17.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.18.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.19.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.20.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.21.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.22.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.23.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.24.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.25.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.26.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.27.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.28.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.29.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.30.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.31.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.32.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.33.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.34.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.35.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.36.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.37.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.38.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.39.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.40.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.41.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.42.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.43.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.44.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.45.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.46.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.47.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.48.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.49.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.50.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.51.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.52.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.53.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.54.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.55.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.56.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.57.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.58.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.59.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.60.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.61.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.62.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.mlp.experts.63.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.self_attn.kv_a_layernorm.weight": "model-00003-of-00005.safetensors", + "model.layers.16.input_layernorm.weight": "model-00003-of-00005.safetensors", + "model.layers.16.post_attention_layernorm.weight": "model-00003-of-00005.safetensors", + "model.layers.16.self_attn.q_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.self_attn.kv_b_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.self_attn.kv_a_proj_with_mqa.weight": "model-00003-of-00005.safetensors", + "model.layers.16.self_attn.o_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.gate.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.shared_experts.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.shared_experts.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.shared_experts.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.gate.e_score_correction_bias": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.0.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.0.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.1.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.1.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.2.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.2.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.3.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.3.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.4.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.4.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.5.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.5.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.6.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.6.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.7.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.7.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.8.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.8.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.9.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.9.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.10.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.10.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.11.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.11.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.12.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.12.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.13.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.13.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.14.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.14.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.15.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.15.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.16.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.16.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.17.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.17.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.18.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.18.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.19.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.19.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.20.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.20.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.21.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.21.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.22.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.22.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.23.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.23.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.24.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.24.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.25.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.25.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.26.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.26.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.27.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.27.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.28.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.28.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.29.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.29.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.30.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.30.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.31.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.31.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.32.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.32.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.33.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.33.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.34.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.34.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.35.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.35.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.36.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.36.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.37.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.37.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.38.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.38.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.39.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.39.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.40.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.40.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.41.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.41.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.42.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.42.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.43.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.43.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.44.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.44.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.45.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.45.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.46.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.46.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.47.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.47.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.48.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.48.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.49.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.49.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.50.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.50.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.51.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.51.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.52.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.52.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.53.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.53.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.54.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.54.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.55.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.55.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.56.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.56.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.57.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.57.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.58.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.58.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.59.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.59.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.60.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.60.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.61.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.61.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.62.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.62.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.63.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.63.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.0.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.1.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.2.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.3.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.4.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.5.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.6.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.7.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.8.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.9.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.10.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.11.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.12.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.13.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.14.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.15.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.16.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.17.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.18.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.19.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.20.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.21.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.22.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.23.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.24.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.25.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.26.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.27.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.28.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.29.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.30.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.31.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.32.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.33.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.34.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.35.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.36.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.37.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.38.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.39.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.40.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.41.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.42.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.43.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.44.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.45.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.46.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.47.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.48.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.49.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.50.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.51.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.52.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.53.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.54.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.55.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.56.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.57.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.58.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.59.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.60.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.61.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.62.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.experts.63.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.self_attn.kv_a_layernorm.weight": "model-00003-of-00005.safetensors", + "model.layers.17.input_layernorm.weight": "model-00003-of-00005.safetensors", + "model.layers.17.post_attention_layernorm.weight": "model-00003-of-00005.safetensors", + "model.layers.17.self_attn.q_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.self_attn.kv_b_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.self_attn.kv_a_proj_with_mqa.weight": "model-00003-of-00005.safetensors", + "model.layers.17.self_attn.o_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.gate.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.shared_experts.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.shared_experts.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.shared_experts.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.gate.e_score_correction_bias": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.0.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.0.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.1.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.1.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.2.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.2.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.3.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.3.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.4.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.4.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.5.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.5.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.6.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.6.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.7.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.7.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.8.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.8.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.9.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.9.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.10.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.10.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.11.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.11.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.12.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.12.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.13.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.13.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.14.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.14.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.15.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.15.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.16.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.16.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.17.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.17.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.18.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.18.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.19.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.19.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.20.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.20.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.21.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.21.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.22.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.22.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.23.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.23.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.24.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.24.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.25.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.25.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.26.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.26.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.27.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.27.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.28.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.28.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.29.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.29.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.30.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.30.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.31.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.31.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.32.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.32.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.33.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.33.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.34.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.34.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.35.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.35.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.36.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.36.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.37.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.37.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.38.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.38.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.39.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.39.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.40.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.40.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.41.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.41.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.42.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.42.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.43.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.43.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.44.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.44.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.45.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.45.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.46.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.46.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.47.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.47.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.48.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.48.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.49.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.49.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.50.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.50.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.51.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.51.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.52.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.52.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.53.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.53.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.54.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.54.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.55.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.55.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.56.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.56.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.57.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.57.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.58.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.58.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.59.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.59.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.60.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.60.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.61.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.61.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.62.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.62.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.63.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.63.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.0.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.1.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.2.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.3.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.4.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.5.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.6.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.7.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.8.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.9.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.10.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.11.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.12.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.13.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.14.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.15.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.16.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.17.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.18.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.19.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.20.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.21.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.22.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.23.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.24.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.25.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.26.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.27.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.28.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.29.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.30.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.31.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.32.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.33.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.34.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.35.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.36.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.37.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.38.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.39.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.40.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.41.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.42.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.43.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.44.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.45.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.46.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.47.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.48.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.49.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.50.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.51.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.52.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.53.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.54.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.55.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.56.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.57.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.58.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.59.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.60.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.61.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.62.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.experts.63.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.self_attn.kv_a_layernorm.weight": "model-00003-of-00005.safetensors", + "model.layers.18.input_layernorm.weight": "model-00003-of-00005.safetensors", + "model.layers.18.post_attention_layernorm.weight": "model-00003-of-00005.safetensors", + "model.layers.18.self_attn.q_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.self_attn.kv_b_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.self_attn.kv_a_proj_with_mqa.weight": "model-00003-of-00005.safetensors", + "model.layers.18.self_attn.o_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.gate.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.shared_experts.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.shared_experts.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.shared_experts.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.gate.e_score_correction_bias": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.0.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.0.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.1.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.1.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.2.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.2.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.3.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.3.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.4.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.4.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.5.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.5.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.6.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.6.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.7.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.7.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.8.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.8.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.9.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.9.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.10.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.10.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.11.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.11.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.12.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.12.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.13.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.13.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.14.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.14.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.15.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.15.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.16.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.16.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.17.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.17.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.18.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.18.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.19.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.19.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.20.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.20.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.21.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.21.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.22.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.22.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.23.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.23.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.24.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.24.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.25.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.25.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.26.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.26.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.27.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.27.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.28.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.28.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.29.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.29.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.30.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.30.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.31.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.31.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.32.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.32.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.33.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.33.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.34.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.34.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.35.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.35.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.36.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.36.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.37.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.37.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.38.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.38.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.39.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.39.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.40.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.40.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.41.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.41.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.42.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.42.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.43.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.43.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.44.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.44.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.45.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.45.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.46.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.46.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.47.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.47.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.48.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.48.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.49.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.49.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.50.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.50.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.51.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.51.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.52.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.52.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.53.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.53.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.54.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.54.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.55.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.55.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.56.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.56.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.57.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.57.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.58.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.58.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.59.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.59.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.60.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.60.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.61.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.61.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.62.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.62.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.63.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.63.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.0.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.1.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.2.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.3.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.4.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.5.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.6.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.7.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.8.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.9.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.10.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.11.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.12.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.13.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.14.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.15.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.16.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.17.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.18.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.19.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.20.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.21.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.22.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.23.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.24.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.25.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.26.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.27.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.28.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.29.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.30.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.31.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.32.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.33.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.34.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.35.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.36.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.37.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.38.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.39.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.40.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.41.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.42.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.43.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.44.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.45.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.46.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.47.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.48.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.49.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.50.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.51.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.52.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.53.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.54.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.55.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.56.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.57.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.58.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.59.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.60.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.61.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.62.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.experts.63.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.19.self_attn.kv_a_layernorm.weight": "model-00004-of-00005.safetensors", + "model.layers.19.input_layernorm.weight": "model-00004-of-00005.safetensors", + "model.layers.19.post_attention_layernorm.weight": "model-00004-of-00005.safetensors", + "model.layers.19.self_attn.q_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.self_attn.kv_b_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.self_attn.kv_a_proj_with_mqa.weight": "model-00004-of-00005.safetensors", + "model.layers.19.self_attn.o_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.gate.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.shared_experts.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.shared_experts.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.shared_experts.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.gate.e_score_correction_bias": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.0.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.0.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.1.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.1.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.2.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.2.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.3.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.3.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.4.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.4.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.5.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.5.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.6.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.6.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.7.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.7.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.8.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.8.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.9.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.9.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.10.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.10.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.11.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.11.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.12.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.12.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.13.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.13.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.14.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.14.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.15.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.15.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.16.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.16.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.17.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.17.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.18.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.18.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.19.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.19.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.20.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.20.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.21.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.21.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.22.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.22.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.23.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.23.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.24.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.24.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.25.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.25.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.26.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.26.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.27.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.27.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.28.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.28.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.29.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.29.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.30.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.30.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.31.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.31.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.32.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.32.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.33.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.33.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.34.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.34.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.35.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.35.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.36.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.36.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.37.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.37.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.38.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.38.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.39.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.39.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.40.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.40.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.41.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.41.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.42.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.42.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.43.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.43.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.44.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.44.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.45.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.45.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.46.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.46.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.47.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.47.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.48.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.48.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.49.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.49.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.50.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.50.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.51.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.51.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.52.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.52.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.53.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.53.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.54.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.54.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.55.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.55.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.56.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.56.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.57.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.57.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.58.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.58.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.59.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.59.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.60.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.60.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.61.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.61.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.62.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.62.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.63.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.63.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.0.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.1.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.2.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.3.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.4.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.5.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.6.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.7.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.8.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.9.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.10.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.11.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.12.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.13.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.14.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.15.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.16.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.17.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.18.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.19.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.20.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.21.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.22.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.23.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.24.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.25.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.26.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.27.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.28.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.29.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.30.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.31.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.32.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.33.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.34.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.35.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.36.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.37.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.38.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.39.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.40.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.41.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.42.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.43.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.44.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.45.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.46.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.47.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.48.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.49.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.50.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.51.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.52.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.53.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.54.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.55.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.56.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.57.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.58.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.59.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.60.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.61.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.62.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.19.mlp.experts.63.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.self_attn.kv_a_layernorm.weight": "model-00004-of-00005.safetensors", + "model.layers.20.input_layernorm.weight": "model-00004-of-00005.safetensors", + "model.layers.20.post_attention_layernorm.weight": "model-00004-of-00005.safetensors", + "model.layers.20.self_attn.q_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.self_attn.kv_b_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.self_attn.kv_a_proj_with_mqa.weight": "model-00004-of-00005.safetensors", + "model.layers.20.self_attn.o_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.gate.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.shared_experts.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.shared_experts.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.shared_experts.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.gate.e_score_correction_bias": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.0.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.0.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.1.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.1.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.2.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.2.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.3.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.3.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.4.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.4.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.5.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.5.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.6.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.6.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.7.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.7.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.8.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.8.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.9.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.9.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.10.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.10.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.11.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.11.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.12.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.12.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.13.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.13.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.14.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.14.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.15.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.15.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.16.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.16.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.17.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.17.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.18.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.18.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.19.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.19.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.20.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.20.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.21.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.21.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.22.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.22.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.23.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.23.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.24.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.24.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.25.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.25.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.26.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.26.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.27.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.27.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.28.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.28.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.29.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.29.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.30.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.30.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.31.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.31.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.32.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.32.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.33.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.33.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.34.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.34.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.35.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.35.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.36.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.36.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.37.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.37.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.38.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.38.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.39.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.39.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.40.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.40.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.41.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.41.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.42.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.42.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.43.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.43.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.44.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.44.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.45.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.45.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.46.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.46.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.47.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.47.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.48.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.48.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.49.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.49.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.50.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.50.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.51.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.51.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.52.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.52.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.53.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.53.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.54.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.54.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.55.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.55.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.56.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.56.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.57.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.57.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.58.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.58.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.59.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.59.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.60.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.60.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.61.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.61.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.62.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.62.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.63.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.63.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.0.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.1.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.2.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.3.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.4.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.5.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.6.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.7.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.8.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.9.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.10.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.11.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.12.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.13.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.14.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.15.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.16.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.17.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.18.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.19.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.20.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.21.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.22.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.23.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.24.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.25.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.26.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.27.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.28.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.29.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.30.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.31.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.32.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.33.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.34.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.35.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.36.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.37.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.38.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.39.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.40.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.41.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.42.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.43.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.44.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.45.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.46.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.47.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.48.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.49.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.50.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.51.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.52.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.53.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.54.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.55.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.56.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.57.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.58.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.59.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.60.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.61.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.62.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.20.mlp.experts.63.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.self_attn.kv_a_layernorm.weight": "model-00004-of-00005.safetensors", + "model.layers.21.input_layernorm.weight": "model-00004-of-00005.safetensors", + "model.layers.21.post_attention_layernorm.weight": "model-00004-of-00005.safetensors", + "model.layers.21.self_attn.q_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.self_attn.kv_b_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.self_attn.kv_a_proj_with_mqa.weight": "model-00004-of-00005.safetensors", + "model.layers.21.self_attn.o_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.gate.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.shared_experts.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.shared_experts.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.shared_experts.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.gate.e_score_correction_bias": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.0.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.0.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.1.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.1.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.2.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.2.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.3.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.3.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.4.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.4.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.5.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.5.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.6.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.6.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.7.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.7.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.8.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.8.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.9.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.9.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.10.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.10.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.11.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.11.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.12.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.12.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.13.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.13.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.14.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.14.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.15.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.15.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.16.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.16.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.17.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.17.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.18.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.18.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.19.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.19.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.20.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.20.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.21.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.21.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.22.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.22.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.23.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.23.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.24.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.24.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.25.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.25.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.26.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.26.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.27.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.27.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.28.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.28.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.29.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.29.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.30.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.30.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.31.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.31.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.32.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.32.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.33.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.33.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.34.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.34.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.35.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.35.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.36.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.36.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.37.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.37.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.38.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.38.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.39.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.39.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.40.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.40.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.41.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.41.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.42.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.42.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.43.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.43.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.44.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.44.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.45.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.45.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.46.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.46.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.47.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.47.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.48.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.48.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.49.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.49.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.50.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.50.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.51.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.51.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.52.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.52.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.53.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.53.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.54.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.54.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.55.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.55.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.56.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.56.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.57.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.57.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.58.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.58.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.59.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.59.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.60.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.60.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.61.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.61.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.62.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.62.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.63.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.63.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.0.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.1.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.2.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.3.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.4.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.5.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.6.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.7.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.8.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.9.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.10.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.11.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.12.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.13.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.14.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.15.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.16.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.17.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.18.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.19.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.20.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.21.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.22.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.23.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.24.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.25.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.26.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.27.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.28.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.29.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.30.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.31.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.32.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.33.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.34.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.35.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.36.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.37.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.38.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.39.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.40.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.41.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.42.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.43.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.44.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.45.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.46.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.47.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.48.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.49.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.50.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.51.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.52.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.53.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.54.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.55.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.56.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.57.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.58.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.59.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.60.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.61.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.62.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.21.mlp.experts.63.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.self_attn.kv_a_layernorm.weight": "model-00004-of-00005.safetensors", + "model.layers.22.input_layernorm.weight": "model-00004-of-00005.safetensors", + "model.layers.22.post_attention_layernorm.weight": "model-00004-of-00005.safetensors", + "model.layers.22.self_attn.q_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.self_attn.kv_b_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.self_attn.kv_a_proj_with_mqa.weight": "model-00004-of-00005.safetensors", + "model.layers.22.self_attn.o_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.gate.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.shared_experts.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.shared_experts.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.shared_experts.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.gate.e_score_correction_bias": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.0.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.0.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.1.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.1.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.2.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.2.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.3.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.3.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.4.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.4.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.5.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.5.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.6.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.6.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.7.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.7.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.8.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.8.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.9.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.9.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.10.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.10.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.11.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.11.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.12.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.12.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.13.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.13.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.14.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.14.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.15.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.15.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.16.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.16.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.17.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.17.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.18.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.18.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.19.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.19.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.20.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.20.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.21.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.21.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.22.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.22.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.23.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.23.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.24.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.24.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.25.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.25.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.26.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.26.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.27.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.27.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.28.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.28.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.29.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.29.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.30.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.30.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.31.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.31.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.32.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.32.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.33.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.33.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.34.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.34.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.35.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.35.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.36.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.36.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.37.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.37.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.38.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.38.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.39.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.39.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.40.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.40.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.41.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.41.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.42.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.42.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.43.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.43.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.44.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.44.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.45.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.45.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.46.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.46.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.47.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.47.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.48.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.48.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.49.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.49.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.50.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.50.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.51.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.51.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.52.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.52.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.53.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.53.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.54.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.54.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.55.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.55.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.56.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.56.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.57.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.57.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.58.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.58.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.59.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.59.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.60.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.60.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.61.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.61.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.62.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.62.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.63.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.63.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.0.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.1.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.2.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.3.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.4.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.5.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.6.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.7.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.8.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.9.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.10.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.11.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.12.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.13.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.14.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.15.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.16.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.17.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.18.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.19.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.20.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.21.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.22.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.23.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.24.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.25.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.26.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.27.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.28.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.29.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.30.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.31.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.32.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.33.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.34.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.35.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.36.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.37.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.38.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.39.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.40.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.41.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.42.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.43.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.44.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.45.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.46.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.47.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.48.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.49.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.50.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.51.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.52.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.53.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.54.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.55.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.56.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.57.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.58.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.59.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.60.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.61.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.62.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.22.mlp.experts.63.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.self_attn.kv_a_layernorm.weight": "model-00004-of-00005.safetensors", + "model.layers.23.input_layernorm.weight": "model-00004-of-00005.safetensors", + "model.layers.23.post_attention_layernorm.weight": "model-00004-of-00005.safetensors", + "model.layers.23.self_attn.q_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.self_attn.kv_b_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.self_attn.kv_a_proj_with_mqa.weight": "model-00004-of-00005.safetensors", + "model.layers.23.self_attn.o_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.gate.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.shared_experts.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.shared_experts.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.shared_experts.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.gate.e_score_correction_bias": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.0.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.0.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.1.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.1.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.2.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.2.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.3.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.3.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.4.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.4.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.5.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.5.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.6.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.6.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.7.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.7.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.8.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.8.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.9.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.9.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.10.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.10.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.11.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.11.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.12.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.12.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.13.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.13.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.14.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.14.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.15.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.15.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.16.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.16.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.17.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.17.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.18.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.18.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.19.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.19.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.20.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.20.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.21.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.21.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.22.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.22.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.23.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.23.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.24.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.24.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.25.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.25.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.26.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.26.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.27.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.27.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.28.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.28.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.29.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.29.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.30.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.30.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.31.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.31.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.32.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.32.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.33.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.33.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.34.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.34.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.35.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.35.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.36.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.36.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.37.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.37.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.38.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.38.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.39.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.39.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.40.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.40.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.41.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.41.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.42.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.42.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.43.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.43.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.44.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.44.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.45.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.45.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.46.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.46.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.47.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.47.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.48.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.48.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.49.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.49.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.50.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.50.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.51.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.51.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.52.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.52.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.53.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.53.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.54.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.54.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.55.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.55.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.56.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.56.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.57.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.57.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.58.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.58.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.59.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.59.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.60.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.60.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.61.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.61.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.62.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.62.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.63.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.63.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.0.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.1.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.2.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.3.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.4.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.5.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.6.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.7.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.8.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.9.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.10.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.11.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.12.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.13.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.14.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.15.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.16.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.17.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.18.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.19.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.20.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.21.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.22.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.23.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.24.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.25.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.26.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.27.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.28.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.29.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.30.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.31.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.32.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.33.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.34.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.35.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.36.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.37.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.38.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.39.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.40.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.41.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.42.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.43.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.44.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.45.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.46.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.47.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.48.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.49.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.50.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.51.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.52.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.53.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.54.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.55.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.56.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.57.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.58.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.59.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.60.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.61.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.62.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.23.mlp.experts.63.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.24.self_attn.kv_a_layernorm.weight": "model-00005-of-00005.safetensors", + "model.layers.24.input_layernorm.weight": "model-00005-of-00005.safetensors", + "model.layers.24.post_attention_layernorm.weight": "model-00005-of-00005.safetensors", + "model.layers.24.self_attn.q_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.self_attn.kv_b_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.self_attn.kv_a_proj_with_mqa.weight": "model-00005-of-00005.safetensors", + "model.layers.24.self_attn.o_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.gate.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.shared_experts.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.shared_experts.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.shared_experts.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.gate.e_score_correction_bias": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.0.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.0.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.1.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.1.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.2.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.2.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.3.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.3.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.4.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.4.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.5.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.5.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.6.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.6.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.7.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.7.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.8.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.8.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.9.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.9.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.10.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.10.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.11.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.11.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.12.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.12.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.13.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.13.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.14.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.14.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.15.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.15.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.16.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.16.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.17.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.17.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.18.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.18.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.19.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.19.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.20.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.20.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.21.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.21.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.22.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.22.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.23.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.23.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.24.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.24.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.25.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.25.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.26.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.26.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.27.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.27.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.28.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.28.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.29.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.29.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.30.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.30.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.31.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.31.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.32.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.32.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.33.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.33.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.34.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.34.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.35.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.35.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.36.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.36.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.37.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.37.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.38.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.38.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.39.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.39.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.40.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.40.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.41.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.41.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.42.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.42.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.43.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.43.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.44.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.44.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.45.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.45.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.46.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.46.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.47.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.47.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.48.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.48.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.49.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.49.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.50.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.50.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.51.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.51.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.52.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.52.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.53.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.53.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.54.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.54.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.55.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.55.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.56.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.56.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.57.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.57.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.58.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.58.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.59.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.59.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.60.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.60.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.61.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.61.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.62.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.62.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.63.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.63.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.0.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.1.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.2.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.3.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.4.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.5.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.6.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.7.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.8.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.9.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.10.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.11.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.12.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.13.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.14.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.15.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.16.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.17.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.18.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.19.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.20.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.21.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.22.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.23.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.24.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.25.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.26.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.27.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.28.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.29.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.30.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.31.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.32.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.33.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.34.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.35.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.36.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.37.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.38.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.39.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.40.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.41.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.42.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.43.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.44.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.45.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.46.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.47.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.48.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.49.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.50.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.51.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.52.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.53.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.54.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.55.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.56.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.57.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.58.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.59.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.60.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.61.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.62.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.24.mlp.experts.63.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.self_attn.kv_a_layernorm.weight": "model-00005-of-00005.safetensors", + "model.layers.25.input_layernorm.weight": "model-00005-of-00005.safetensors", + "model.layers.25.post_attention_layernorm.weight": "model-00005-of-00005.safetensors", + "model.layers.25.self_attn.q_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.self_attn.kv_b_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.self_attn.kv_a_proj_with_mqa.weight": "model-00005-of-00005.safetensors", + "model.layers.25.self_attn.o_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.gate.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.shared_experts.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.shared_experts.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.shared_experts.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.gate.e_score_correction_bias": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.0.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.0.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.1.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.1.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.2.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.2.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.3.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.3.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.4.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.4.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.5.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.5.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.6.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.6.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.7.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.7.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.8.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.8.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.9.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.9.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.10.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.10.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.11.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.11.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.12.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.12.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.13.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.13.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.14.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.14.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.15.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.15.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.16.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.16.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.17.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.17.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.18.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.18.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.19.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.19.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.20.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.20.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.21.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.21.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.22.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.22.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.23.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.23.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.24.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.24.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.25.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.25.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.26.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.26.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.27.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.27.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.28.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.28.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.29.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.29.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.30.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.30.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.31.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.31.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.32.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.32.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.33.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.33.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.34.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.34.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.35.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.35.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.36.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.36.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.37.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.37.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.38.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.38.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.39.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.39.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.40.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.40.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.41.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.41.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.42.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.42.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.43.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.43.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.44.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.44.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.45.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.45.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.46.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.46.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.47.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.47.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.48.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.48.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.49.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.49.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.50.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.50.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.51.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.51.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.52.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.52.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.53.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.53.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.54.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.54.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.55.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.55.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.56.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.56.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.57.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.57.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.58.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.58.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.59.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.59.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.60.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.60.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.61.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.61.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.62.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.62.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.63.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.63.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.0.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.1.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.2.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.3.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.4.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.5.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.6.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.7.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.8.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.9.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.10.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.11.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.12.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.13.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.14.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.15.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.16.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.17.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.18.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.19.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.20.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.21.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.22.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.23.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.24.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.25.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.26.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.27.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.28.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.29.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.30.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.31.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.32.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.33.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.34.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.35.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.36.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.37.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.38.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.39.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.40.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.41.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.42.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.43.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.44.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.45.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.46.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.47.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.48.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.49.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.50.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.51.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.52.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.53.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.54.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.55.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.56.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.57.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.58.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.59.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.60.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.61.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.62.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.25.mlp.experts.63.down_proj.weight": "model-00005-of-00005.safetensors" + } +} \ No newline at end of file diff --git a/special_tokens_map.json b/special_tokens_map.json new file mode 100644 index 0000000..d91fcde --- /dev/null +++ b/special_tokens_map.json @@ -0,0 +1,16 @@ +{ + "bos_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } +} diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..5b0cce9 --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b4b3d90c67830a4e566296ad8d0f6b5ac5a5cdbfd331b200d0ee8263aaaea1fe +size 10680800 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..c99094c --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,131 @@ +{ + "added_tokens_decoder": { + "1": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "2": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128000": { + "content": "<|role_sep|>\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128001": { + "content": "<|message_sep|>\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128002": { + "content": "<|file|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128003": { + "content": "<|/file|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128004": { + "content": "[image_token]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128005": { + "content": "[video_image_token]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128006": { + "content": "[audio_token]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128007": { + "content": "[video_audio_token]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128008": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128009": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128010": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128011": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128012": { + "content": "<|function_call|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "bos_token": "", + "clean_up_tokenization_spaces": true, + "eos_token": "", + "extra_special_tokens": {}, + "model_max_length": 1000000000000000019884624838656, + "tokenizer_class": "PreTrainedTokenizer", + "unk_token": null +}