初始化项目，由ModelHub XC社区提供模型

Model: huihui-ai/Huihui-granite-4.1-3b-abliterated
Source: Original Platform

.gitattributes

@@ -0,0 +1,36 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+ggml-model-bf16.gguf filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,250 @@
+---
+license: apache-2.0
+library_name: transformers
+base_model:
+- ibm-granite/granite-4.1-3b
+tags:
+- language
+- granite-4.1
+- abliterated
+- uncensored
+---
+
+# huihui-ai/Huihui-granite-4.1-3b-abliterated
+
+
+This is an uncensored version of [ibm-granite/granite-4.1-3b](https://huggingface.co/ibm-granite/granite-4.1-3b) created with abliteration (see [remove-refusals-with-transformers](https://github.com/Sumandora/remove-refusals-with-transformers) to know more about it).
+This is a crude, proof-of-concept implementation to remove refusals from an LLM model without using TransformerLens.
+
+## ollama
+
+Please use the latest version of [ollama](https://github.com/ollama/ollama/releases/tag)
+
+You can use [huihui_ai/granite4.1-abliterated:3b](https://ollama.com/huihui_ai/granite4.1-abliterated:3b) directly, 
+
+```
+ollama run huihui_ai/granite4.1-abliterated:3b
+```
+
+
+## Usage
+You can use this model in your applications by loading it with Hugging Face's `transformers` library:
+
+
+```python
+
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import argparse
+from transformers import AutoModelForCausalLM, AutoTokenizer,  TextStreamer
+import torch
+import os
+import signal
+import time
+
+def parse_args():
+    parser = argparse.ArgumentParser(
+        description="Merge LoRA weights into huihui-ai/Huihui-granite-4.1-3b-abliterated base model and save the full model."
+    )
+    parser.add_argument(
+        "--base_model",
+        type=str,
+        default="huihui-ai/Huihui-granite-4.1-3b-abliterated",
+        help="HuggingFace repo or local path of the base model.",
+    )
+    parser.add_argument(
+        "--dtype",
+        type=str,
+        default="bfloat16",
+        choices=["auto", "float16", "bfloat16", "float32"],
+        help="Data type for loading the base model (default: bfloat16).",
+    )
+    parser.add_argument(
+        "--device_map",
+        type=str,
+        default="auto",
+        help="Device map for model loading (e.g. 'cpu', 'auto').",
+    )
+    return parser.parse_args()
+
+def main():
+    cpu_count = os.cpu_count()
+    print(f"Number of CPU cores in the system: {cpu_count}")
+    half_cpu_count = cpu_count // 2
+    os.environ["MKL_NUM_THREADS"] = str(half_cpu_count)
+    os.environ["OMP_NUM_THREADS"] = str(half_cpu_count)
+    torch.set_num_threads(half_cpu_count)
+
+    print(f"PyTorch threads: {torch.get_num_threads()}")
+    print(f"MKL threads: {os.getenv('MKL_NUM_THREADS')}")
+    print(f"OMP threads: {os.getenv('OMP_NUM_THREADS')}")
+
+    args = parse_args()
+
+    # Load the model and tokenizer
+    print(f"Load Model {args.base_model} ... ")
+
+    torch_dtype = {
+        "auto": "auto",
+        "float16": torch.float16,
+        "bfloat16": torch.bfloat16,
+        "float32": torch.float32,
+    }[args.dtype]
+
+    model = AutoModelForCausalLM.from_pretrained(
+        args.base_model,
+        dtype=torch_dtype,
+        device_map=args.device_map,
+        trust_remote_code=True,
+    )
+
+    tokenizer = AutoTokenizer.from_pretrained(args.base_model)
+
+    class CustomTextStreamer(TextStreamer):
+        def __init__(self, tokenizer, skip_prompt=True, skip_special_tokens=True):
+            super().__init__(tokenizer, skip_prompt=skip_prompt, skip_special_tokens=skip_special_tokens)
+            self.generated_text = ""
+            self.stop_flag = False
+            self.init_time = time.time()  # Record initialization time
+            self.end_time = None  # To store end time
+            self.first_token_time = None  # To store first token generation time
+            self.token_count = 0  # To track total tokens
+
+        def on_finalized_text(self, text: str, stream_end: bool = False):
+            if self.first_token_time is None and text.strip():  # Set first token time on first non-empty text
+                self.first_token_time = time.time()
+            if stream_end:
+                self.end_time = time.time()  # Record end time when streaming ends
+
+            self.generated_text += text
+            self.token_count += 1
+            print(text, end="", flush=True)
+
+            if self.stop_flag:
+                raise StopIteration
+
+        def stop_generation(self):
+            self.stop_flag = True
+            self.end_time = time.time()  # Record end time when generation is stopped
+
+        def get_metrics(self):
+            """Returns initialization time, first token time, first token latency, end time, total time, total tokens, and tokens per second."""
+            if self.end_time is None:
+                self.end_time = time.time()  # Set end time if not already set
+            total_time = self.end_time - self.init_time  # Total time from init to end
+            tokens_per_second = self.token_count / total_time if total_time > 0 else 0
+            first_token_latency = (self.first_token_time - self.init_time) if self.first_token_time is not None else None
+            metrics = {
+                "init_time": self.init_time,
+                "first_token_time": self.first_token_time,
+                "first_token_latency": first_token_latency,
+                "end_time": self.end_time,
+                "total_time": total_time,  # Total time in seconds
+                "total_tokens": self.token_count,
+                "tokens_per_second": tokens_per_second
+            }
+            return metrics
+
+    def generate_stream(model, tokenizer, messages, enable_thinking, skip_prompt, skip_special_tokens, max_new_tokens):
+        text = tokenizer.apply_chat_template(
+            messages,
+            tokenize=False,
+            add_generation_prompt=True,
+        )
+        inputs = tokenizer(
+            text,
+            return_tensors="pt",
+        ).to(model.device)
+
+        streamer = CustomTextStreamer(tokenizer, skip_prompt=skip_prompt, skip_special_tokens=skip_special_tokens)
+
+        def signal_handler(sig, frame):
+            streamer.stop_generation()
+            print("\n[Generation stopped by user with Ctrl+C]")
+
+        signal.signal(signal.SIGINT, signal_handler)
+
+        print("Response: ", end="", flush=True)
+        try:
+            generated_ids = model.generate(
+                **inputs,
+                max_new_tokens=max_new_tokens,
+                streamer=streamer)
+
+            del generated_ids
+        except StopIteration:
+            print("\n[Stopped by user]")
+
+        del inputs
+        torch.cuda.empty_cache()
+        signal.signal(signal.SIGINT, signal.SIG_DFL)
+
+        return streamer.generated_text, streamer.stop_flag, streamer.get_metrics()
+
+    messages = []
+    skip_prompt=True
+    skip_special_tokens=True
+
+    while True:
+        print(f"skip_prompt = {skip_prompt}.")
+        print(f"skip_special_tokens = {skip_special_tokens}.\n")
+
+        user_input = input("User: ").strip()
+        if user_input.lower() == "/exit":
+            print("Exiting chat.")
+            break
+        if user_input.lower() == "/clear":
+            messages = []
+            print("Chat history cleared. Starting a new conversation.")
+            continue
+        if user_input.lower() == "/skip_prompt":
+            skip_prompt = not skip_prompt
+            continue
+        if user_input.lower() == "/skip_special_tokens":
+            skip_special_tokens = not skip_special_tokens
+            continue
+        if not user_input:
+            print("Input cannot be empty. Please enter something.")
+            continue
+
+        messages.append({"role": "user", "content": user_input})
+        response, stop_flag, metrics = generate_stream(model, tokenizer, messages, enable_thinking, skip_prompt, skip_special_tokens, 40960)
+        print("\n\nMetrics:")
+        for key, value in metrics.items():
+            print(f"  {key}: {value}")
+
+        print("", flush=True)
+
+        if stop_flag:
+            continue
+        messages.append({"role": "assistant", "content": response})
+
+if __name__ == "__main__":
+    main()
+```
+
+### Usage Warnings
+
+
+ - **Risk of Sensitive or Controversial Outputs**: This model’s safety filtering has been significantly reduced, potentially generating sensitive, controversial, or inappropriate content. Users should exercise caution and rigorously review generated outputs.
+
+ - **Not Suitable for All Audiences**: Due to limited content filtering, the model’s outputs may be inappropriate for public settings, underage users, or applications requiring high security.
+
+ - **Legal and Ethical Responsibilities**: Users must ensure their usage complies with local laws and ethical standards. Generated content may carry legal or ethical risks, and users are solely responsible for any consequences.
+
+ - **Research and Experimental Use**: It is recommended to use this model for research, testing, or controlled environments, avoiding direct use in production or public-facing commercial applications.
+
+ - **Monitoring and Review Recommendations**: Users are strongly advised to monitor model outputs in real-time and conduct manual reviews when necessary to prevent the dissemination of inappropriate content.
+
+ - **No Default Safety Guarantees**: Unlike standard models, this model has not undergone rigorous safety optimization. huihui.ai bears no responsibility for any consequences arising from its use.
+
+
+### Donation
+##### Your donation helps us continue our further development and improvement, a cup of coffee can do it.
+- bitcoin:
+```
+  bc1qqnkhuchxw0zqjh2ku3lu4hq45hc6gy84uk70ge
+```
+- Support our work on [Ko-fi](https://ko-fi.com/huihuiai)!
+

chat_template.jinja

@@ -0,0 +1,114 @@
+{%- set tools_system_message_prefix = 'You are a helpful assistant with access to the following tools. You may call one or more tools to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>'  %}
+{%- set tools_system_message_suffix = '\n</tools>\n\nFor each tool call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call>. If a tool does not exist in the provided list of tools, notify the user that you do not have the ability to fulfill the request.' %}
+{%- set documents_system_message_prefix = 'You are a helpful assistant with access to the following documents. You may use one or more documents to assist with the user query.\n\nYou are given a list of documents within <documents></documents> XML tags:\n<documents>' %}
+{%- set documents_system_message_suffix = '\n</documents>\n\nWrite the response to the user\'s input by strictly aligning with the facts in the provided documents. If the information needed to answer the question is not available in the documents, inform the user that the question cannot be answered based on the available data.' %}
+{%- if available_tools is defined and available_tools %}
+    {%- set tools = available_tools %}
+{%- endif %}
+{%- set ns = namespace(tools_system_message=tools_system_message_prefix,
+                       documents_system_message=documents_system_message_prefix,
+                       system_message=''
+                       ) %}
+{%- if tools %}
+    {%- for tool in tools %}
+        {%- set ns.tools_system_message = ns.tools_system_message + '\n' + (tool | tojson) %}
+    {%- endfor %}
+    {%- set ns.tools_system_message = ns.tools_system_message + tools_system_message_suffix %}
+{%- else %}
+    {%- set ns.tools_system_message = '' %}
+{%- endif %}
+{%- if documents %}
+    {%- for document in documents %}
+        {%- set ns.documents_system_message = ns.documents_system_message + '\n' + (document | tojson) %}
+    {%- endfor %}
+    {%- set ns.documents_system_message = ns.documents_system_message + documents_system_message_suffix %}
+{%- else %}
+    {%- set ns.documents_system_message = '' %}
+{%- endif %}
+{%- if messages[0].role == 'system' %}
+    {%- if messages[0].content is string %}
+        {%- set ns.system_message = messages[0].content %}
+    {%- elif messages[0].content is iterable %}
+        {%- for entry in messages[0].content %}
+            {%- if entry.type== 'text' %}
+                {%- if ns.system_message != '' %}
+                    {%- set ns.system_message = ns.system_message + '\n' %}
+                {%- endif %}
+                {%- set ns.system_message = ns.system_message + entry.text %}
+            {%- endif %}
+        {%- endfor %}
+    {%- endif %}
+    {%- if tools and documents %}
+        {%- set ns.system_message = ns.system_message + '\n\n' +  ns.tools_system_message + '\n\n' + ns.documents_system_message %}
+    {%- elif tools %}
+        {%- set ns.system_message = ns.system_message + '\n\n' + ns.tools_system_message %}
+    {%- elif documents %}
+        {%- set ns.system_message = ns.system_message + '\n\n' + ns.documents_system_message %}
+    {%- endif %}
+{%- else %}
+    {%- if tools and documents %}
+        {%- set ns.system_message = ns.tools_system_message + '\n\n' + ns.documents_system_message  %}
+    {%- elif tools %}
+        {%- set ns.system_message = ns.tools_system_message %}
+    {%- elif documents %}
+        {%- set ns.system_message = ns.documents_system_message %}
+    {%- endif %}
+{%- endif %}
+{%- if ns.system_message %}
+    {{- '<|start_of_role|>system<|end_of_role|>' + ns.system_message + '<|end_of_text|>\n' }}
+{%- endif %}
+{%- for message in messages %}
+    {%- set content = namespace(val='') %}
+    {%- if message.content is string %}
+        {%- set content.val = message.content %}
+    {%- else %}
+        {%- if message.content is iterable %}
+            {%- for entry in message.content %}
+                {%- if entry.type== 'text' %}
+                    {%- if content.val != '' %}
+                        {%- set content.val = content.val + '\n' %}
+                    {%- endif %}
+                    {%- set content.val = content.val + entry.text %}
+                {%- endif %}
+            {%- endfor %}
+        {%- endif %}
+    {%- endif %}
+    {%- if (message.role == 'user') or (message.role == 'system' and not loop.first) %}
+        {{- '<|start_of_role|>' + message.role + '<|end_of_role|>' + content.val + '<|end_of_text|>\n' }}
+    {%- elif message.role == 'assistant' %}
+        {{- '<|start_of_role|>' + message.role + '<|end_of_role|>' + content.val }}
+        {%- if message.tool_calls %}
+            {%- for tool_call in message.tool_calls %}
+                {%- if (loop.first and content.val) or (not loop.first) %}
+                    {{- '\n' }}
+                {%- endif %}
+                {%- if tool_call.function %}
+                    {%- set tool_call = tool_call.function %}
+                {%- endif %}
+                {{- '<tool_call>\n{"name": "' }}
+                {{- tool_call.name }}
+                {{- '", "arguments": ' }}
+                {%- if tool_call.arguments is string %}
+                    {{- tool_call.arguments }}
+                {%- else %}
+                    {{- tool_call.arguments | tojson }}
+                {%- endif %}
+                {{- '}\n</tool_call>' }}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|end_of_text|>\n' }}
+    {%- elif message.role == 'tool' %}
+        {%- if loop.first or (messages[loop.index0 - 1].role != 'tool') %}
+            {{- '<|start_of_role|>user<|end_of_role|>' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- content.val }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != 'tool') %}
+            {{- '<|end_of_text|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|start_of_role|>assistant<|end_of_role|>' }}
+{%- endif %}

config.json

@@ -0,0 +1,32 @@
+{
+  "architectures": [
+    "GraniteForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "attention_multiplier": 0.015625,
+  "bos_token_id": 100257,
+  "embedding_multiplier": 12.0,
+  "eos_token_id": 100257,
+  "hidden_act": "silu",
+  "hidden_size": 2560,
+  "initializer_range": 0.1,
+  "intermediate_size": 8192,
+  "logits_scaling": 10.0,
+  "max_position_embeddings": 131072,
+  "mlp_bias": false,
+  "model_type": "granite",
+  "num_attention_heads": 40,
+  "num_hidden_layers": 40,
+  "num_key_value_heads": 8,
+  "pad_token_id": 100256,
+  "residual_multiplier": 0.22,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": null,
+  "rope_theta": 10000000,
+  "tie_word_embeddings": true,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.53.3",
+  "use_cache": true,
+  "vocab_size": 100352
+}

generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 100257,
+  "eos_token_id": 100257,
+  "pad_token_id": 100256,
+  "transformers_version": "4.53.3"
+}

ggml-model-bf16.gguf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:12d6f2128e9aa113c1c6ee6a9a24678dc53fccbb3e84bca5165c58ea5f4b14cc
+size 6809656064

model-00001-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6e427afab9e8f757332e398a5b846e5d181ccfc3801e19e2f7273c91717aec66
+size 4991538312

model-00002-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fa1d23fe546c6bf9220555d9aaccf23b7fd475cb968ebed10d52a8fe693dfdf5
+size 1814176416

model.safetensors.index.json

@@ -0,0 +1,370 @@
+{
+  "metadata": {
+    "total_parameters": 3402836480,
+    "total_size": 6805672960
+  },
+  "weight_map": {
+    "model.embed_tokens.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.32.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.32.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.32.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.32.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.32.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.32.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.32.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.32.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.32.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.33.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.33.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.33.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.33.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.33.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.33.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.33.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.33.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.33.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.34.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.34.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.34.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.34.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.34.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.34.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.34.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.34.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.34.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.35.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.35.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.35.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.35.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.35.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.35.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.35.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.35.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.35.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.36.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.36.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.36.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.36.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.36.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.36.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.36.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.36.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.36.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.37.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.37.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.37.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.37.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.37.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.37.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.37.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.37.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.37.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.38.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.38.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.38.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.38.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.38.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.38.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.38.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.38.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.38.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.39.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.39.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.39.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.39.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.39.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.39.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.39.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.39.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.39.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.norm.weight": "model-00002-of-00002.safetensors"
+  }
+}

model_params1.txt

@@ -0,0 +1,429 @@
+ibm-granite/granite-4.1-3b
+------------------------------------------------------------
+GraniteForCausalLM(
+  (model): GraniteModel(
+    (embed_tokens): Embedding(100352, 2560, padding_idx=100256)
+    (layers): ModuleList(
+      (0-39): 40 x GraniteDecoderLayer(
+        (self_attn): GraniteAttention(
+          (q_proj): Linear(in_features=2560, out_features=2560, bias=False)
+          (k_proj): Linear(in_features=2560, out_features=512, bias=False)
+          (v_proj): Linear(in_features=2560, out_features=512, bias=False)
+          (o_proj): Linear(in_features=2560, out_features=2560, bias=False)
+        )
+        (mlp): GraniteMLP(
+          (gate_proj): Linear(in_features=2560, out_features=8192, bias=False)
+          (up_proj): Linear(in_features=2560, out_features=8192, bias=False)
+          (down_proj): Linear(in_features=8192, out_features=2560, bias=False)
+          (act_fn): SiLUActivation()
+        )
+        (input_layernorm): GraniteRMSNorm((2560,), eps=1e-05)
+        (post_attention_layernorm): GraniteRMSNorm((2560,), eps=1e-05)
+      )
+    )
+    (norm): GraniteRMSNorm((2560,), eps=1e-05)
+    (rotary_emb): GraniteRotaryEmbedding()
+  )
+  (lm_head): Linear(in_features=2560, out_features=100352, bias=False)
+)
+------------------------------------------------------------
+GraniteConfig {
+  "architectures": [
+    "GraniteForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "attention_multiplier": 0.015625,
+  "bos_token_id": 100257,
+  "dtype": "bfloat16",
+  "embedding_multiplier": 12.0,
+  "eos_token_id": 100257,
+  "hidden_act": "silu",
+  "hidden_size": 2560,
+  "initializer_range": 0.1,
+  "intermediate_size": 8192,
+  "logits_scaling": 10.0,
+  "max_position_embeddings": 131072,
+  "mlp_bias": false,
+  "model_type": "granite",
+  "num_attention_heads": 40,
+  "num_hidden_layers": 40,
+  "num_key_value_heads": 8,
+  "pad_token_id": 100256,
+  "residual_multiplier": 0.22,
+  "rms_norm_eps": 1e-05,
+  "rope_parameters": {
+    "rope_theta": 10000000,
+    "rope_type": "default"
+  },
+  "tie_word_embeddings": true,
+  "transformers_version": "5.6.2",
+  "use_cache": true,
+  "vocab_size": 100352
+}
+
+------------------------------------------------------------
+model.embed_tokens.weight: dtype:torch.bfloat16, shape:torch.Size([100352, 2560]), size:256,901,120, device: cuda:0
+model.layers.0.self_attn.q_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 2560]), size:6,553,600, device: cuda:0
+model.layers.0.self_attn.k_proj.weight: dtype:torch.bfloat16, shape:torch.Size([512, 2560]), size:1,310,720, device: cuda:0
+model.layers.0.self_attn.v_proj.weight: dtype:torch.bfloat16, shape:torch.Size([512, 2560]), size:1,310,720, device: cuda:0
+model.layers.0.self_attn.o_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 2560]), size:6,553,600, device: cuda:0
+model.layers.0.mlp.gate_proj.weight: dtype:torch.bfloat16, shape:torch.Size([8192, 2560]), size:20,971,520, device: cuda:0
+model.layers.0.mlp.up_proj.weight: dtype:torch.bfloat16, shape:torch.Size([8192, 2560]), size:20,971,520, device: cuda:0
+model.layers.0.mlp.down_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 8192]), size:20,971,520, device: cuda:0
+model.layers.0.input_layernorm.weight: dtype:torch.bfloat16, shape:torch.Size([2560]), size:2,560, device: cuda:0
+model.layers.0.post_attention_layernorm.weight: dtype:torch.bfloat16, shape:torch.Size([2560]), size:2,560, device: cuda:0
+model.layers.1.self_attn.q_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 2560]), size:6,553,600, device: cuda:0
+model.layers.1.self_attn.k_proj.weight: dtype:torch.bfloat16, shape:torch.Size([512, 2560]), size:1,310,720, device: cuda:0
+model.layers.1.self_attn.v_proj.weight: dtype:torch.bfloat16, shape:torch.Size([512, 2560]), size:1,310,720, device: cuda:0
+model.layers.1.self_attn.o_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 2560]), size:6,553,600, device: cuda:0
+model.layers.1.mlp.gate_proj.weight: dtype:torch.bfloat16, shape:torch.Size([8192, 2560]), size:20,971,520, device: cuda:0
+model.layers.1.mlp.up_proj.weight: dtype:torch.bfloat16, shape:torch.Size([8192, 2560]), size:20,971,520, device: cuda:0
+model.layers.1.mlp.down_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 8192]), size:20,971,520, device: cuda:0
+model.layers.1.input_layernorm.weight: dtype:torch.bfloat16, shape:torch.Size([2560]), size:2,560, device: cuda:0
+model.layers.1.post_attention_layernorm.weight: dtype:torch.bfloat16, shape:torch.Size([2560]), size:2,560, device: cuda:0
+model.layers.2.self_attn.q_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 2560]), size:6,553,600, device: cuda:0
+model.layers.2.self_attn.k_proj.weight: dtype:torch.bfloat16, shape:torch.Size([512, 2560]), size:1,310,720, device: cuda:0
+model.layers.2.self_attn.v_proj.weight: dtype:torch.bfloat16, shape:torch.Size([512, 2560]), size:1,310,720, device: cuda:0
+model.layers.2.self_attn.o_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 2560]), size:6,553,600, device: cuda:0
+model.layers.2.mlp.gate_proj.weight: dtype:torch.bfloat16, shape:torch.Size([8192, 2560]), size:20,971,520, device: cuda:0
+model.layers.2.mlp.up_proj.weight: dtype:torch.bfloat16, shape:torch.Size([8192, 2560]), size:20,971,520, device: cuda:0
+model.layers.2.mlp.down_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 8192]), size:20,971,520, device: cuda:0
+model.layers.2.input_layernorm.weight: dtype:torch.bfloat16, shape:torch.Size([2560]), size:2,560, device: cuda:0
+model.layers.2.post_attention_layernorm.weight: dtype:torch.bfloat16, shape:torch.Size([2560]), size:2,560, device: cuda:0
+model.layers.3.self_attn.q_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 2560]), size:6,553,600, device: cuda:0
+model.layers.3.self_attn.k_proj.weight: dtype:torch.bfloat16, shape:torch.Size([512, 2560]), size:1,310,720, device: cuda:0
+model.layers.3.self_attn.v_proj.weight: dtype:torch.bfloat16, shape:torch.Size([512, 2560]), size:1,310,720, device: cuda:0
+model.layers.3.self_attn.o_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 2560]), size:6,553,600, device: cuda:0
+model.layers.3.mlp.gate_proj.weight: dtype:torch.bfloat16, shape:torch.Size([8192, 2560]), size:20,971,520, device: cuda:0
+model.layers.3.mlp.up_proj.weight: dtype:torch.bfloat16, shape:torch.Size([8192, 2560]), size:20,971,520, device: cuda:0
+model.layers.3.mlp.down_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 8192]), size:20,971,520, device: cuda:0
+model.layers.3.input_layernorm.weight: dtype:torch.bfloat16, shape:torch.Size([2560]), size:2,560, device: cuda:0
+model.layers.3.post_attention_layernorm.weight: dtype:torch.bfloat16, shape:torch.Size([2560]), size:2,560, device: cuda:0
+model.layers.4.self_attn.q_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 2560]), size:6,553,600, device: cuda:0
+model.layers.4.self_attn.k_proj.weight: dtype:torch.bfloat16, shape:torch.Size([512, 2560]), size:1,310,720, device: cuda:0
+model.layers.4.self_attn.v_proj.weight: dtype:torch.bfloat16, shape:torch.Size([512, 2560]), size:1,310,720, device: cuda:0
+model.layers.4.self_attn.o_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 2560]), size:6,553,600, device: cuda:0
+model.layers.4.mlp.gate_proj.weight: dtype:torch.bfloat16, shape:torch.Size([8192, 2560]), size:20,971,520, device: cuda:0
+model.layers.4.mlp.up_proj.weight: dtype:torch.bfloat16, shape:torch.Size([8192, 2560]), size:20,971,520, device: cuda:0
+model.layers.4.mlp.down_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 8192]), size:20,971,520, device: cuda:0
+model.layers.4.input_layernorm.weight: dtype:torch.bfloat16, shape:torch.Size([2560]), size:2,560, device: cuda:0
+model.layers.4.post_attention_layernorm.weight: dtype:torch.bfloat16, shape:torch.Size([2560]), size:2,560, device: cuda:0
+model.layers.5.self_attn.q_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 2560]), size:6,553,600, device: cuda:0
+model.layers.5.self_attn.k_proj.weight: dtype:torch.bfloat16, shape:torch.Size([512, 2560]), size:1,310,720, device: cuda:0
+model.layers.5.self_attn.v_proj.weight: dtype:torch.bfloat16, shape:torch.Size([512, 2560]), size:1,310,720, device: cuda:0
+model.layers.5.self_attn.o_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 2560]), size:6,553,600, device: cuda:0
+model.layers.5.mlp.gate_proj.weight: dtype:torch.bfloat16, shape:torch.Size([8192, 2560]), size:20,971,520, device: cuda:0
+model.layers.5.mlp.up_proj.weight: dtype:torch.bfloat16, shape:torch.Size([8192, 2560]), size:20,971,520, device: cuda:0
+model.layers.5.mlp.down_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 8192]), size:20,971,520, device: cuda:0
+model.layers.5.input_layernorm.weight: dtype:torch.bfloat16, shape:torch.Size([2560]), size:2,560, device: cuda:0
+model.layers.5.post_attention_layernorm.weight: dtype:torch.bfloat16, shape:torch.Size([2560]), size:2,560, device: cuda:0
+model.layers.6.self_attn.q_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 2560]), size:6,553,600, device: cuda:0
+model.layers.6.self_attn.k_proj.weight: dtype:torch.bfloat16, shape:torch.Size([512, 2560]), size:1,310,720, device: cuda:0
+model.layers.6.self_attn.v_proj.weight: dtype:torch.bfloat16, shape:torch.Size([512, 2560]), size:1,310,720, device: cuda:0
+model.layers.6.self_attn.o_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 2560]), size:6,553,600, device: cuda:0
+model.layers.6.mlp.gate_proj.weight: dtype:torch.bfloat16, shape:torch.Size([8192, 2560]), size:20,971,520, device: cuda:0
+model.layers.6.mlp.up_proj.weight: dtype:torch.bfloat16, shape:torch.Size([8192, 2560]), size:20,971,520, device: cuda:0
+model.layers.6.mlp.down_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 8192]), size:20,971,520, device: cuda:0
+model.layers.6.input_layernorm.weight: dtype:torch.bfloat16, shape:torch.Size([2560]), size:2,560, device: cuda:0
+model.layers.6.post_attention_layernorm.weight: dtype:torch.bfloat16, shape:torch.Size([2560]), size:2,560, device: cuda:0
+model.layers.7.self_attn.q_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 2560]), size:6,553,600, device: cuda:0
+model.layers.7.self_attn.k_proj.weight: dtype:torch.bfloat16, shape:torch.Size([512, 2560]), size:1,310,720, device: cuda:0
+model.layers.7.self_attn.v_proj.weight: dtype:torch.bfloat16, shape:torch.Size([512, 2560]), size:1,310,720, device: cuda:0
+model.layers.7.self_attn.o_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 2560]), size:6,553,600, device: cuda:0
+model.layers.7.mlp.gate_proj.weight: dtype:torch.bfloat16, shape:torch.Size([8192, 2560]), size:20,971,520, device: cuda:0
+model.layers.7.mlp.up_proj.weight: dtype:torch.bfloat16, shape:torch.Size([8192, 2560]), size:20,971,520, device: cuda:0
+model.layers.7.mlp.down_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 8192]), size:20,971,520, device: cuda:0
+model.layers.7.input_layernorm.weight: dtype:torch.bfloat16, shape:torch.Size([2560]), size:2,560, device: cuda:0
+model.layers.7.post_attention_layernorm.weight: dtype:torch.bfloat16, shape:torch.Size([2560]), size:2,560, device: cuda:0
+model.layers.8.self_attn.q_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 2560]), size:6,553,600, device: cuda:0
+model.layers.8.self_attn.k_proj.weight: dtype:torch.bfloat16, shape:torch.Size([512, 2560]), size:1,310,720, device: cuda:0
+model.layers.8.self_attn.v_proj.weight: dtype:torch.bfloat16, shape:torch.Size([512, 2560]), size:1,310,720, device: cuda:0
+model.layers.8.self_attn.o_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 2560]), size:6,553,600, device: cuda:0
+model.layers.8.mlp.gate_proj.weight: dtype:torch.bfloat16, shape:torch.Size([8192, 2560]), size:20,971,520, device: cuda:0
+model.layers.8.mlp.up_proj.weight: dtype:torch.bfloat16, shape:torch.Size([8192, 2560]), size:20,971,520, device: cuda:0
+model.layers.8.mlp.down_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 8192]), size:20,971,520, device: cuda:0
+model.layers.8.input_layernorm.weight: dtype:torch.bfloat16, shape:torch.Size([2560]), size:2,560, device: cuda:0
+model.layers.8.post_attention_layernorm.weight: dtype:torch.bfloat16, shape:torch.Size([2560]), size:2,560, device: cuda:0
+model.layers.9.self_attn.q_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 2560]), size:6,553,600, device: cuda:0
+model.layers.9.self_attn.k_proj.weight: dtype:torch.bfloat16, shape:torch.Size([512, 2560]), size:1,310,720, device: cuda:0
+model.layers.9.self_attn.v_proj.weight: dtype:torch.bfloat16, shape:torch.Size([512, 2560]), size:1,310,720, device: cuda:0
+model.layers.9.self_attn.o_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 2560]), size:6,553,600, device: cuda:0
+model.layers.9.mlp.gate_proj.weight: dtype:torch.bfloat16, shape:torch.Size([8192, 2560]), size:20,971,520, device: cuda:0
+model.layers.9.mlp.up_proj.weight: dtype:torch.bfloat16, shape:torch.Size([8192, 2560]), size:20,971,520, device: cuda:0
+model.layers.9.mlp.down_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 8192]), size:20,971,520, device: cuda:0
+model.layers.9.input_layernorm.weight: dtype:torch.bfloat16, shape:torch.Size([2560]), size:2,560, device: cuda:0
+model.layers.9.post_attention_layernorm.weight: dtype:torch.bfloat16, shape:torch.Size([2560]), size:2,560, device: cuda:0
+model.layers.10.self_attn.q_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 2560]), size:6,553,600, device: cuda:0
+model.layers.10.self_attn.k_proj.weight: dtype:torch.bfloat16, shape:torch.Size([512, 2560]), size:1,310,720, device: cuda:0
+model.layers.10.self_attn.v_proj.weight: dtype:torch.bfloat16, shape:torch.Size([512, 2560]), size:1,310,720, device: cuda:0
+model.layers.10.self_attn.o_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 2560]), size:6,553,600, device: cuda:0
+model.layers.10.mlp.gate_proj.weight: dtype:torch.bfloat16, shape:torch.Size([8192, 2560]), size:20,971,520, device: cuda:0
+model.layers.10.mlp.up_proj.weight: dtype:torch.bfloat16, shape:torch.Size([8192, 2560]), size:20,971,520, device: cuda:0
+model.layers.10.mlp.down_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 8192]), size:20,971,520, device: cuda:0
+model.layers.10.input_layernorm.weight: dtype:torch.bfloat16, shape:torch.Size([2560]), size:2,560, device: cuda:0
+model.layers.10.post_attention_layernorm.weight: dtype:torch.bfloat16, shape:torch.Size([2560]), size:2,560, device: cuda:0
+model.layers.11.self_attn.q_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 2560]), size:6,553,600, device: cuda:0
+model.layers.11.self_attn.k_proj.weight: dtype:torch.bfloat16, shape:torch.Size([512, 2560]), size:1,310,720, device: cuda:0
+model.layers.11.self_attn.v_proj.weight: dtype:torch.bfloat16, shape:torch.Size([512, 2560]), size:1,310,720, device: cuda:0
+model.layers.11.self_attn.o_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 2560]), size:6,553,600, device: cuda:0
+model.layers.11.mlp.gate_proj.weight: dtype:torch.bfloat16, shape:torch.Size([8192, 2560]), size:20,971,520, device: cuda:0
+model.layers.11.mlp.up_proj.weight: dtype:torch.bfloat16, shape:torch.Size([8192, 2560]), size:20,971,520, device: cuda:0
+model.layers.11.mlp.down_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 8192]), size:20,971,520, device: cuda:0
+model.layers.11.input_layernorm.weight: dtype:torch.bfloat16, shape:torch.Size([2560]), size:2,560, device: cuda:0
+model.layers.11.post_attention_layernorm.weight: dtype:torch.bfloat16, shape:torch.Size([2560]), size:2,560, device: cuda:0
+model.layers.12.self_attn.q_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 2560]), size:6,553,600, device: cuda:0
+model.layers.12.self_attn.k_proj.weight: dtype:torch.bfloat16, shape:torch.Size([512, 2560]), size:1,310,720, device: cuda:0
+model.layers.12.self_attn.v_proj.weight: dtype:torch.bfloat16, shape:torch.Size([512, 2560]), size:1,310,720, device: cuda:0
+model.layers.12.self_attn.o_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 2560]), size:6,553,600, device: cuda:0
+model.layers.12.mlp.gate_proj.weight: dtype:torch.bfloat16, shape:torch.Size([8192, 2560]), size:20,971,520, device: cuda:0
+model.layers.12.mlp.up_proj.weight: dtype:torch.bfloat16, shape:torch.Size([8192, 2560]), size:20,971,520, device: cuda:0
+model.layers.12.mlp.down_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 8192]), size:20,971,520, device: cuda:0
+model.layers.12.input_layernorm.weight: dtype:torch.bfloat16, shape:torch.Size([2560]), size:2,560, device: cuda:0
+model.layers.12.post_attention_layernorm.weight: dtype:torch.bfloat16, shape:torch.Size([2560]), size:2,560, device: cuda:0
+model.layers.13.self_attn.q_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 2560]), size:6,553,600, device: cuda:0
+model.layers.13.self_attn.k_proj.weight: dtype:torch.bfloat16, shape:torch.Size([512, 2560]), size:1,310,720, device: cuda:0
+model.layers.13.self_attn.v_proj.weight: dtype:torch.bfloat16, shape:torch.Size([512, 2560]), size:1,310,720, device: cuda:0
+model.layers.13.self_attn.o_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 2560]), size:6,553,600, device: cuda:0
+model.layers.13.mlp.gate_proj.weight: dtype:torch.bfloat16, shape:torch.Size([8192, 2560]), size:20,971,520, device: cuda:0
+model.layers.13.mlp.up_proj.weight: dtype:torch.bfloat16, shape:torch.Size([8192, 2560]), size:20,971,520, device: cuda:0
+model.layers.13.mlp.down_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 8192]), size:20,971,520, device: cuda:0
+model.layers.13.input_layernorm.weight: dtype:torch.bfloat16, shape:torch.Size([2560]), size:2,560, device: cuda:0
+model.layers.13.post_attention_layernorm.weight: dtype:torch.bfloat16, shape:torch.Size([2560]), size:2,560, device: cuda:0
+model.layers.14.self_attn.q_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 2560]), size:6,553,600, device: cuda:0
+model.layers.14.self_attn.k_proj.weight: dtype:torch.bfloat16, shape:torch.Size([512, 2560]), size:1,310,720, device: cuda:0
+model.layers.14.self_attn.v_proj.weight: dtype:torch.bfloat16, shape:torch.Size([512, 2560]), size:1,310,720, device: cuda:0
+model.layers.14.self_attn.o_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 2560]), size:6,553,600, device: cuda:0
+model.layers.14.mlp.gate_proj.weight: dtype:torch.bfloat16, shape:torch.Size([8192, 2560]), size:20,971,520, device: cuda:0
+model.layers.14.mlp.up_proj.weight: dtype:torch.bfloat16, shape:torch.Size([8192, 2560]), size:20,971,520, device: cuda:0
+model.layers.14.mlp.down_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 8192]), size:20,971,520, device: cuda:0
+model.layers.14.input_layernorm.weight: dtype:torch.bfloat16, shape:torch.Size([2560]), size:2,560, device: cuda:0
+model.layers.14.post_attention_layernorm.weight: dtype:torch.bfloat16, shape:torch.Size([2560]), size:2,560, device: cuda:0
+model.layers.15.self_attn.q_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 2560]), size:6,553,600, device: cuda:0
+model.layers.15.self_attn.k_proj.weight: dtype:torch.bfloat16, shape:torch.Size([512, 2560]), size:1,310,720, device: cuda:0
+model.layers.15.self_attn.v_proj.weight: dtype:torch.bfloat16, shape:torch.Size([512, 2560]), size:1,310,720, device: cuda:0
+model.layers.15.self_attn.o_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 2560]), size:6,553,600, device: cuda:0
+model.layers.15.mlp.gate_proj.weight: dtype:torch.bfloat16, shape:torch.Size([8192, 2560]), size:20,971,520, device: cuda:0
+model.layers.15.mlp.up_proj.weight: dtype:torch.bfloat16, shape:torch.Size([8192, 2560]), size:20,971,520, device: cuda:0
+model.layers.15.mlp.down_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 8192]), size:20,971,520, device: cuda:0
+model.layers.15.input_layernorm.weight: dtype:torch.bfloat16, shape:torch.Size([2560]), size:2,560, device: cuda:0
+model.layers.15.post_attention_layernorm.weight: dtype:torch.bfloat16, shape:torch.Size([2560]), size:2,560, device: cuda:0
+model.layers.16.self_attn.q_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 2560]), size:6,553,600, device: cuda:0
+model.layers.16.self_attn.k_proj.weight: dtype:torch.bfloat16, shape:torch.Size([512, 2560]), size:1,310,720, device: cuda:0
+model.layers.16.self_attn.v_proj.weight: dtype:torch.bfloat16, shape:torch.Size([512, 2560]), size:1,310,720, device: cuda:0
+model.layers.16.self_attn.o_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 2560]), size:6,553,600, device: cuda:0
+model.layers.16.mlp.gate_proj.weight: dtype:torch.bfloat16, shape:torch.Size([8192, 2560]), size:20,971,520, device: cuda:0
+model.layers.16.mlp.up_proj.weight: dtype:torch.bfloat16, shape:torch.Size([8192, 2560]), size:20,971,520, device: cuda:0
+model.layers.16.mlp.down_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 8192]), size:20,971,520, device: cuda:0
+model.layers.16.input_layernorm.weight: dtype:torch.bfloat16, shape:torch.Size([2560]), size:2,560, device: cuda:0
+model.layers.16.post_attention_layernorm.weight: dtype:torch.bfloat16, shape:torch.Size([2560]), size:2,560, device: cuda:0
+model.layers.17.self_attn.q_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 2560]), size:6,553,600, device: cuda:0
+model.layers.17.self_attn.k_proj.weight: dtype:torch.bfloat16, shape:torch.Size([512, 2560]), size:1,310,720, device: cuda:0
+model.layers.17.self_attn.v_proj.weight: dtype:torch.bfloat16, shape:torch.Size([512, 2560]), size:1,310,720, device: cuda:0
+model.layers.17.self_attn.o_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 2560]), size:6,553,600, device: cuda:0
+model.layers.17.mlp.gate_proj.weight: dtype:torch.bfloat16, shape:torch.Size([8192, 2560]), size:20,971,520, device: cuda:0
+model.layers.17.mlp.up_proj.weight: dtype:torch.bfloat16, shape:torch.Size([8192, 2560]), size:20,971,520, device: cuda:0
+model.layers.17.mlp.down_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 8192]), size:20,971,520, device: cuda:0
+model.layers.17.input_layernorm.weight: dtype:torch.bfloat16, shape:torch.Size([2560]), size:2,560, device: cuda:0
+model.layers.17.post_attention_layernorm.weight: dtype:torch.bfloat16, shape:torch.Size([2560]), size:2,560, device: cuda:0
+model.layers.18.self_attn.q_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 2560]), size:6,553,600, device: cuda:0
+model.layers.18.self_attn.k_proj.weight: dtype:torch.bfloat16, shape:torch.Size([512, 2560]), size:1,310,720, device: cuda:0
+model.layers.18.self_attn.v_proj.weight: dtype:torch.bfloat16, shape:torch.Size([512, 2560]), size:1,310,720, device: cuda:0
+model.layers.18.self_attn.o_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 2560]), size:6,553,600, device: cuda:0
+model.layers.18.mlp.gate_proj.weight: dtype:torch.bfloat16, shape:torch.Size([8192, 2560]), size:20,971,520, device: cuda:0
+model.layers.18.mlp.up_proj.weight: dtype:torch.bfloat16, shape:torch.Size([8192, 2560]), size:20,971,520, device: cuda:0
+model.layers.18.mlp.down_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 8192]), size:20,971,520, device: cuda:0
+model.layers.18.input_layernorm.weight: dtype:torch.bfloat16, shape:torch.Size([2560]), size:2,560, device: cuda:0
+model.layers.18.post_attention_layernorm.weight: dtype:torch.bfloat16, shape:torch.Size([2560]), size:2,560, device: cuda:0
+model.layers.19.self_attn.q_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 2560]), size:6,553,600, device: cuda:0
+model.layers.19.self_attn.k_proj.weight: dtype:torch.bfloat16, shape:torch.Size([512, 2560]), size:1,310,720, device: cuda:0
+model.layers.19.self_attn.v_proj.weight: dtype:torch.bfloat16, shape:torch.Size([512, 2560]), size:1,310,720, device: cuda:0
+model.layers.19.self_attn.o_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 2560]), size:6,553,600, device: cuda:0
+model.layers.19.mlp.gate_proj.weight: dtype:torch.bfloat16, shape:torch.Size([8192, 2560]), size:20,971,520, device: cuda:0
+model.layers.19.mlp.up_proj.weight: dtype:torch.bfloat16, shape:torch.Size([8192, 2560]), size:20,971,520, device: cuda:0
+model.layers.19.mlp.down_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 8192]), size:20,971,520, device: cuda:0
+model.layers.19.input_layernorm.weight: dtype:torch.bfloat16, shape:torch.Size([2560]), size:2,560, device: cuda:0
+model.layers.19.post_attention_layernorm.weight: dtype:torch.bfloat16, shape:torch.Size([2560]), size:2,560, device: cuda:0
+model.layers.20.self_attn.q_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 2560]), size:6,553,600, device: cuda:0
+model.layers.20.self_attn.k_proj.weight: dtype:torch.bfloat16, shape:torch.Size([512, 2560]), size:1,310,720, device: cuda:0
+model.layers.20.self_attn.v_proj.weight: dtype:torch.bfloat16, shape:torch.Size([512, 2560]), size:1,310,720, device: cuda:0
+model.layers.20.self_attn.o_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 2560]), size:6,553,600, device: cuda:0
+model.layers.20.mlp.gate_proj.weight: dtype:torch.bfloat16, shape:torch.Size([8192, 2560]), size:20,971,520, device: cuda:0
+model.layers.20.mlp.up_proj.weight: dtype:torch.bfloat16, shape:torch.Size([8192, 2560]), size:20,971,520, device: cuda:0
+model.layers.20.mlp.down_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 8192]), size:20,971,520, device: cuda:0
+model.layers.20.input_layernorm.weight: dtype:torch.bfloat16, shape:torch.Size([2560]), size:2,560, device: cuda:0
+model.layers.20.post_attention_layernorm.weight: dtype:torch.bfloat16, shape:torch.Size([2560]), size:2,560, device: cuda:0
+model.layers.21.self_attn.q_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 2560]), size:6,553,600, device: cuda:0
+model.layers.21.self_attn.k_proj.weight: dtype:torch.bfloat16, shape:torch.Size([512, 2560]), size:1,310,720, device: cuda:0
+model.layers.21.self_attn.v_proj.weight: dtype:torch.bfloat16, shape:torch.Size([512, 2560]), size:1,310,720, device: cuda:0
+model.layers.21.self_attn.o_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 2560]), size:6,553,600, device: cuda:0
+model.layers.21.mlp.gate_proj.weight: dtype:torch.bfloat16, shape:torch.Size([8192, 2560]), size:20,971,520, device: cuda:0
+model.layers.21.mlp.up_proj.weight: dtype:torch.bfloat16, shape:torch.Size([8192, 2560]), size:20,971,520, device: cuda:0
+model.layers.21.mlp.down_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 8192]), size:20,971,520, device: cuda:0
+model.layers.21.input_layernorm.weight: dtype:torch.bfloat16, shape:torch.Size([2560]), size:2,560, device: cuda:0
+model.layers.21.post_attention_layernorm.weight: dtype:torch.bfloat16, shape:torch.Size([2560]), size:2,560, device: cuda:0
+model.layers.22.self_attn.q_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 2560]), size:6,553,600, device: cuda:0
+model.layers.22.self_attn.k_proj.weight: dtype:torch.bfloat16, shape:torch.Size([512, 2560]), size:1,310,720, device: cuda:0
+model.layers.22.self_attn.v_proj.weight: dtype:torch.bfloat16, shape:torch.Size([512, 2560]), size:1,310,720, device: cuda:0
+model.layers.22.self_attn.o_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 2560]), size:6,553,600, device: cuda:0
+model.layers.22.mlp.gate_proj.weight: dtype:torch.bfloat16, shape:torch.Size([8192, 2560]), size:20,971,520, device: cuda:0
+model.layers.22.mlp.up_proj.weight: dtype:torch.bfloat16, shape:torch.Size([8192, 2560]), size:20,971,520, device: cuda:0
+model.layers.22.mlp.down_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 8192]), size:20,971,520, device: cuda:0
+model.layers.22.input_layernorm.weight: dtype:torch.bfloat16, shape:torch.Size([2560]), size:2,560, device: cuda:0
+model.layers.22.post_attention_layernorm.weight: dtype:torch.bfloat16, shape:torch.Size([2560]), size:2,560, device: cuda:0
+model.layers.23.self_attn.q_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 2560]), size:6,553,600, device: cuda:0
+model.layers.23.self_attn.k_proj.weight: dtype:torch.bfloat16, shape:torch.Size([512, 2560]), size:1,310,720, device: cuda:0
+model.layers.23.self_attn.v_proj.weight: dtype:torch.bfloat16, shape:torch.Size([512, 2560]), size:1,310,720, device: cuda:0
+model.layers.23.self_attn.o_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 2560]), size:6,553,600, device: cuda:0
+model.layers.23.mlp.gate_proj.weight: dtype:torch.bfloat16, shape:torch.Size([8192, 2560]), size:20,971,520, device: cuda:0
+model.layers.23.mlp.up_proj.weight: dtype:torch.bfloat16, shape:torch.Size([8192, 2560]), size:20,971,520, device: cuda:0
+model.layers.23.mlp.down_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 8192]), size:20,971,520, device: cuda:0
+model.layers.23.input_layernorm.weight: dtype:torch.bfloat16, shape:torch.Size([2560]), size:2,560, device: cuda:0
+model.layers.23.post_attention_layernorm.weight: dtype:torch.bfloat16, shape:torch.Size([2560]), size:2,560, device: cuda:0
+model.layers.24.self_attn.q_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 2560]), size:6,553,600, device: cuda:0
+model.layers.24.self_attn.k_proj.weight: dtype:torch.bfloat16, shape:torch.Size([512, 2560]), size:1,310,720, device: cuda:0
+model.layers.24.self_attn.v_proj.weight: dtype:torch.bfloat16, shape:torch.Size([512, 2560]), size:1,310,720, device: cuda:0
+model.layers.24.self_attn.o_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 2560]), size:6,553,600, device: cuda:0
+model.layers.24.mlp.gate_proj.weight: dtype:torch.bfloat16, shape:torch.Size([8192, 2560]), size:20,971,520, device: cuda:0
+model.layers.24.mlp.up_proj.weight: dtype:torch.bfloat16, shape:torch.Size([8192, 2560]), size:20,971,520, device: cuda:0
+model.layers.24.mlp.down_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 8192]), size:20,971,520, device: cuda:0
+model.layers.24.input_layernorm.weight: dtype:torch.bfloat16, shape:torch.Size([2560]), size:2,560, device: cuda:0
+model.layers.24.post_attention_layernorm.weight: dtype:torch.bfloat16, shape:torch.Size([2560]), size:2,560, device: cuda:0
+model.layers.25.self_attn.q_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 2560]), size:6,553,600, device: cuda:0
+model.layers.25.self_attn.k_proj.weight: dtype:torch.bfloat16, shape:torch.Size([512, 2560]), size:1,310,720, device: cuda:0
+model.layers.25.self_attn.v_proj.weight: dtype:torch.bfloat16, shape:torch.Size([512, 2560]), size:1,310,720, device: cuda:0
+model.layers.25.self_attn.o_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 2560]), size:6,553,600, device: cuda:0
+model.layers.25.mlp.gate_proj.weight: dtype:torch.bfloat16, shape:torch.Size([8192, 2560]), size:20,971,520, device: cuda:0
+model.layers.25.mlp.up_proj.weight: dtype:torch.bfloat16, shape:torch.Size([8192, 2560]), size:20,971,520, device: cuda:0
+model.layers.25.mlp.down_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 8192]), size:20,971,520, device: cuda:0
+model.layers.25.input_layernorm.weight: dtype:torch.bfloat16, shape:torch.Size([2560]), size:2,560, device: cuda:0
+model.layers.25.post_attention_layernorm.weight: dtype:torch.bfloat16, shape:torch.Size([2560]), size:2,560, device: cuda:0
+model.layers.26.self_attn.q_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 2560]), size:6,553,600, device: cuda:0
+model.layers.26.self_attn.k_proj.weight: dtype:torch.bfloat16, shape:torch.Size([512, 2560]), size:1,310,720, device: cuda:0
+model.layers.26.self_attn.v_proj.weight: dtype:torch.bfloat16, shape:torch.Size([512, 2560]), size:1,310,720, device: cuda:0
+model.layers.26.self_attn.o_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 2560]), size:6,553,600, device: cuda:0
+model.layers.26.mlp.gate_proj.weight: dtype:torch.bfloat16, shape:torch.Size([8192, 2560]), size:20,971,520, device: cuda:0
+model.layers.26.mlp.up_proj.weight: dtype:torch.bfloat16, shape:torch.Size([8192, 2560]), size:20,971,520, device: cuda:0
+model.layers.26.mlp.down_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 8192]), size:20,971,520, device: cuda:0
+model.layers.26.input_layernorm.weight: dtype:torch.bfloat16, shape:torch.Size([2560]), size:2,560, device: cuda:0
+model.layers.26.post_attention_layernorm.weight: dtype:torch.bfloat16, shape:torch.Size([2560]), size:2,560, device: cuda:0
+model.layers.27.self_attn.q_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 2560]), size:6,553,600, device: cuda:0
+model.layers.27.self_attn.k_proj.weight: dtype:torch.bfloat16, shape:torch.Size([512, 2560]), size:1,310,720, device: cuda:0
+model.layers.27.self_attn.v_proj.weight: dtype:torch.bfloat16, shape:torch.Size([512, 2560]), size:1,310,720, device: cuda:0
+model.layers.27.self_attn.o_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 2560]), size:6,553,600, device: cuda:0
+model.layers.27.mlp.gate_proj.weight: dtype:torch.bfloat16, shape:torch.Size([8192, 2560]), size:20,971,520, device: cuda:0
+model.layers.27.mlp.up_proj.weight: dtype:torch.bfloat16, shape:torch.Size([8192, 2560]), size:20,971,520, device: cuda:0
+model.layers.27.mlp.down_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 8192]), size:20,971,520, device: cuda:0
+model.layers.27.input_layernorm.weight: dtype:torch.bfloat16, shape:torch.Size([2560]), size:2,560, device: cuda:0
+model.layers.27.post_attention_layernorm.weight: dtype:torch.bfloat16, shape:torch.Size([2560]), size:2,560, device: cuda:0
+model.layers.28.self_attn.q_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 2560]), size:6,553,600, device: cuda:0
+model.layers.28.self_attn.k_proj.weight: dtype:torch.bfloat16, shape:torch.Size([512, 2560]), size:1,310,720, device: cuda:0
+model.layers.28.self_attn.v_proj.weight: dtype:torch.bfloat16, shape:torch.Size([512, 2560]), size:1,310,720, device: cuda:0
+model.layers.28.self_attn.o_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 2560]), size:6,553,600, device: cuda:0
+model.layers.28.mlp.gate_proj.weight: dtype:torch.bfloat16, shape:torch.Size([8192, 2560]), size:20,971,520, device: cuda:0
+model.layers.28.mlp.up_proj.weight: dtype:torch.bfloat16, shape:torch.Size([8192, 2560]), size:20,971,520, device: cuda:0
+model.layers.28.mlp.down_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 8192]), size:20,971,520, device: cuda:0
+model.layers.28.input_layernorm.weight: dtype:torch.bfloat16, shape:torch.Size([2560]), size:2,560, device: cuda:0
+model.layers.28.post_attention_layernorm.weight: dtype:torch.bfloat16, shape:torch.Size([2560]), size:2,560, device: cuda:0
+model.layers.29.self_attn.q_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 2560]), size:6,553,600, device: cuda:0
+model.layers.29.self_attn.k_proj.weight: dtype:torch.bfloat16, shape:torch.Size([512, 2560]), size:1,310,720, device: cuda:0
+model.layers.29.self_attn.v_proj.weight: dtype:torch.bfloat16, shape:torch.Size([512, 2560]), size:1,310,720, device: cuda:0
+model.layers.29.self_attn.o_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 2560]), size:6,553,600, device: cuda:0
+model.layers.29.mlp.gate_proj.weight: dtype:torch.bfloat16, shape:torch.Size([8192, 2560]), size:20,971,520, device: cuda:0
+model.layers.29.mlp.up_proj.weight: dtype:torch.bfloat16, shape:torch.Size([8192, 2560]), size:20,971,520, device: cuda:0
+model.layers.29.mlp.down_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 8192]), size:20,971,520, device: cuda:0
+model.layers.29.input_layernorm.weight: dtype:torch.bfloat16, shape:torch.Size([2560]), size:2,560, device: cuda:0
+model.layers.29.post_attention_layernorm.weight: dtype:torch.bfloat16, shape:torch.Size([2560]), size:2,560, device: cuda:0
+model.layers.30.self_attn.q_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 2560]), size:6,553,600, device: cuda:0
+model.layers.30.self_attn.k_proj.weight: dtype:torch.bfloat16, shape:torch.Size([512, 2560]), size:1,310,720, device: cuda:0
+model.layers.30.self_attn.v_proj.weight: dtype:torch.bfloat16, shape:torch.Size([512, 2560]), size:1,310,720, device: cuda:0
+model.layers.30.self_attn.o_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 2560]), size:6,553,600, device: cuda:0
+model.layers.30.mlp.gate_proj.weight: dtype:torch.bfloat16, shape:torch.Size([8192, 2560]), size:20,971,520, device: cuda:0
+model.layers.30.mlp.up_proj.weight: dtype:torch.bfloat16, shape:torch.Size([8192, 2560]), size:20,971,520, device: cuda:0
+model.layers.30.mlp.down_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 8192]), size:20,971,520, device: cuda:0
+model.layers.30.input_layernorm.weight: dtype:torch.bfloat16, shape:torch.Size([2560]), size:2,560, device: cuda:0
+model.layers.30.post_attention_layernorm.weight: dtype:torch.bfloat16, shape:torch.Size([2560]), size:2,560, device: cuda:0
+model.layers.31.self_attn.q_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 2560]), size:6,553,600, device: cuda:0
+model.layers.31.self_attn.k_proj.weight: dtype:torch.bfloat16, shape:torch.Size([512, 2560]), size:1,310,720, device: cuda:0
+model.layers.31.self_attn.v_proj.weight: dtype:torch.bfloat16, shape:torch.Size([512, 2560]), size:1,310,720, device: cuda:0
+model.layers.31.self_attn.o_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 2560]), size:6,553,600, device: cuda:0
+model.layers.31.mlp.gate_proj.weight: dtype:torch.bfloat16, shape:torch.Size([8192, 2560]), size:20,971,520, device: cuda:0
+model.layers.31.mlp.up_proj.weight: dtype:torch.bfloat16, shape:torch.Size([8192, 2560]), size:20,971,520, device: cuda:0
+model.layers.31.mlp.down_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 8192]), size:20,971,520, device: cuda:0
+model.layers.31.input_layernorm.weight: dtype:torch.bfloat16, shape:torch.Size([2560]), size:2,560, device: cuda:0
+model.layers.31.post_attention_layernorm.weight: dtype:torch.bfloat16, shape:torch.Size([2560]), size:2,560, device: cuda:0
+model.layers.32.self_attn.q_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 2560]), size:6,553,600, device: cuda:0
+model.layers.32.self_attn.k_proj.weight: dtype:torch.bfloat16, shape:torch.Size([512, 2560]), size:1,310,720, device: cuda:0
+model.layers.32.self_attn.v_proj.weight: dtype:torch.bfloat16, shape:torch.Size([512, 2560]), size:1,310,720, device: cuda:0
+model.layers.32.self_attn.o_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 2560]), size:6,553,600, device: cuda:0
+model.layers.32.mlp.gate_proj.weight: dtype:torch.bfloat16, shape:torch.Size([8192, 2560]), size:20,971,520, device: cuda:0
+model.layers.32.mlp.up_proj.weight: dtype:torch.bfloat16, shape:torch.Size([8192, 2560]), size:20,971,520, device: cuda:0
+model.layers.32.mlp.down_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 8192]), size:20,971,520, device: cuda:0
+model.layers.32.input_layernorm.weight: dtype:torch.bfloat16, shape:torch.Size([2560]), size:2,560, device: cuda:0
+model.layers.32.post_attention_layernorm.weight: dtype:torch.bfloat16, shape:torch.Size([2560]), size:2,560, device: cuda:0
+model.layers.33.self_attn.q_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 2560]), size:6,553,600, device: cuda:0
+model.layers.33.self_attn.k_proj.weight: dtype:torch.bfloat16, shape:torch.Size([512, 2560]), size:1,310,720, device: cuda:0
+model.layers.33.self_attn.v_proj.weight: dtype:torch.bfloat16, shape:torch.Size([512, 2560]), size:1,310,720, device: cuda:0
+model.layers.33.self_attn.o_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 2560]), size:6,553,600, device: cuda:0
+model.layers.33.mlp.gate_proj.weight: dtype:torch.bfloat16, shape:torch.Size([8192, 2560]), size:20,971,520, device: cuda:0
+model.layers.33.mlp.up_proj.weight: dtype:torch.bfloat16, shape:torch.Size([8192, 2560]), size:20,971,520, device: cuda:0
+model.layers.33.mlp.down_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 8192]), size:20,971,520, device: cuda:0
+model.layers.33.input_layernorm.weight: dtype:torch.bfloat16, shape:torch.Size([2560]), size:2,560, device: cuda:0
+model.layers.33.post_attention_layernorm.weight: dtype:torch.bfloat16, shape:torch.Size([2560]), size:2,560, device: cuda:0
+model.layers.34.self_attn.q_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 2560]), size:6,553,600, device: cuda:0
+model.layers.34.self_attn.k_proj.weight: dtype:torch.bfloat16, shape:torch.Size([512, 2560]), size:1,310,720, device: cuda:0
+model.layers.34.self_attn.v_proj.weight: dtype:torch.bfloat16, shape:torch.Size([512, 2560]), size:1,310,720, device: cuda:0
+model.layers.34.self_attn.o_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 2560]), size:6,553,600, device: cuda:0
+model.layers.34.mlp.gate_proj.weight: dtype:torch.bfloat16, shape:torch.Size([8192, 2560]), size:20,971,520, device: cuda:0
+model.layers.34.mlp.up_proj.weight: dtype:torch.bfloat16, shape:torch.Size([8192, 2560]), size:20,971,520, device: cuda:0
+model.layers.34.mlp.down_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 8192]), size:20,971,520, device: cuda:0
+model.layers.34.input_layernorm.weight: dtype:torch.bfloat16, shape:torch.Size([2560]), size:2,560, device: cuda:0
+model.layers.34.post_attention_layernorm.weight: dtype:torch.bfloat16, shape:torch.Size([2560]), size:2,560, device: cuda:0
+model.layers.35.self_attn.q_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 2560]), size:6,553,600, device: cuda:0
+model.layers.35.self_attn.k_proj.weight: dtype:torch.bfloat16, shape:torch.Size([512, 2560]), size:1,310,720, device: cuda:0
+model.layers.35.self_attn.v_proj.weight: dtype:torch.bfloat16, shape:torch.Size([512, 2560]), size:1,310,720, device: cuda:0
+model.layers.35.self_attn.o_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 2560]), size:6,553,600, device: cuda:0
+model.layers.35.mlp.gate_proj.weight: dtype:torch.bfloat16, shape:torch.Size([8192, 2560]), size:20,971,520, device: cuda:0
+model.layers.35.mlp.up_proj.weight: dtype:torch.bfloat16, shape:torch.Size([8192, 2560]), size:20,971,520, device: cuda:0
+model.layers.35.mlp.down_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 8192]), size:20,971,520, device: cuda:0
+model.layers.35.input_layernorm.weight: dtype:torch.bfloat16, shape:torch.Size([2560]), size:2,560, device: cuda:0
+model.layers.35.post_attention_layernorm.weight: dtype:torch.bfloat16, shape:torch.Size([2560]), size:2,560, device: cuda:0
+model.layers.36.self_attn.q_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 2560]), size:6,553,600, device: cuda:0
+model.layers.36.self_attn.k_proj.weight: dtype:torch.bfloat16, shape:torch.Size([512, 2560]), size:1,310,720, device: cuda:0
+model.layers.36.self_attn.v_proj.weight: dtype:torch.bfloat16, shape:torch.Size([512, 2560]), size:1,310,720, device: cuda:0
+model.layers.36.self_attn.o_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 2560]), size:6,553,600, device: cuda:0
+model.layers.36.mlp.gate_proj.weight: dtype:torch.bfloat16, shape:torch.Size([8192, 2560]), size:20,971,520, device: cuda:0
+model.layers.36.mlp.up_proj.weight: dtype:torch.bfloat16, shape:torch.Size([8192, 2560]), size:20,971,520, device: cuda:0
+model.layers.36.mlp.down_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 8192]), size:20,971,520, device: cuda:0
+model.layers.36.input_layernorm.weight: dtype:torch.bfloat16, shape:torch.Size([2560]), size:2,560, device: cuda:0
+model.layers.36.post_attention_layernorm.weight: dtype:torch.bfloat16, shape:torch.Size([2560]), size:2,560, device: cuda:0
+model.layers.37.self_attn.q_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 2560]), size:6,553,600, device: cuda:0
+model.layers.37.self_attn.k_proj.weight: dtype:torch.bfloat16, shape:torch.Size([512, 2560]), size:1,310,720, device: cuda:0
+model.layers.37.self_attn.v_proj.weight: dtype:torch.bfloat16, shape:torch.Size([512, 2560]), size:1,310,720, device: cuda:0
+model.layers.37.self_attn.o_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 2560]), size:6,553,600, device: cuda:0
+model.layers.37.mlp.gate_proj.weight: dtype:torch.bfloat16, shape:torch.Size([8192, 2560]), size:20,971,520, device: cuda:0
+model.layers.37.mlp.up_proj.weight: dtype:torch.bfloat16, shape:torch.Size([8192, 2560]), size:20,971,520, device: cuda:0
+model.layers.37.mlp.down_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 8192]), size:20,971,520, device: cuda:0
+model.layers.37.input_layernorm.weight: dtype:torch.bfloat16, shape:torch.Size([2560]), size:2,560, device: cuda:0
+model.layers.37.post_attention_layernorm.weight: dtype:torch.bfloat16, shape:torch.Size([2560]), size:2,560, device: cuda:0
+model.layers.38.self_attn.q_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 2560]), size:6,553,600, device: cuda:0
+model.layers.38.self_attn.k_proj.weight: dtype:torch.bfloat16, shape:torch.Size([512, 2560]), size:1,310,720, device: cuda:0
+model.layers.38.self_attn.v_proj.weight: dtype:torch.bfloat16, shape:torch.Size([512, 2560]), size:1,310,720, device: cuda:0
+model.layers.38.self_attn.o_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 2560]), size:6,553,600, device: cuda:0
+model.layers.38.mlp.gate_proj.weight: dtype:torch.bfloat16, shape:torch.Size([8192, 2560]), size:20,971,520, device: cuda:0
+model.layers.38.mlp.up_proj.weight: dtype:torch.bfloat16, shape:torch.Size([8192, 2560]), size:20,971,520, device: cuda:0
+model.layers.38.mlp.down_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 8192]), size:20,971,520, device: cuda:0
+model.layers.38.input_layernorm.weight: dtype:torch.bfloat16, shape:torch.Size([2560]), size:2,560, device: cuda:0
+model.layers.38.post_attention_layernorm.weight: dtype:torch.bfloat16, shape:torch.Size([2560]), size:2,560, device: cuda:0
+model.layers.39.self_attn.q_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 2560]), size:6,553,600, device: cuda:0
+model.layers.39.self_attn.k_proj.weight: dtype:torch.bfloat16, shape:torch.Size([512, 2560]), size:1,310,720, device: cuda:0
+model.layers.39.self_attn.v_proj.weight: dtype:torch.bfloat16, shape:torch.Size([512, 2560]), size:1,310,720, device: cuda:0
+model.layers.39.self_attn.o_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 2560]), size:6,553,600, device: cuda:0
+model.layers.39.mlp.gate_proj.weight: dtype:torch.bfloat16, shape:torch.Size([8192, 2560]), size:20,971,520, device: cuda:0
+model.layers.39.mlp.up_proj.weight: dtype:torch.bfloat16, shape:torch.Size([8192, 2560]), size:20,971,520, device: cuda:0
+model.layers.39.mlp.down_proj.weight: dtype:torch.bfloat16, shape:torch.Size([2560, 8192]), size:20,971,520, device: cuda:0
+model.layers.39.input_layernorm.weight: dtype:torch.bfloat16, shape:torch.Size([2560]), size:2,560, device: cuda:0
+model.layers.39.post_attention_layernorm.weight: dtype:torch.bfloat16, shape:torch.Size([2560]), size:2,560, device: cuda:0
+model.norm.weight: dtype:torch.bfloat16, shape:torch.Size([2560]), size:2,560, device: cuda:0
+------------------------------------------------------------
+total_params: 3,402,836,480

sharded_ablate.log

@@ -0,0 +1,362 @@
+4, model.embed_tokens.weight, torch.Size([100352, 2560]), layer -1, model-00001-of-00002.safetensors
+3, model.layers.0.input_layernorm.weight, torch.Size([2560]), layer 0, model-00001-of-00002.safetensors
+2, model.layers.0.mlp.down_proj.weight, torch.Size([2560, 8192]), layer 0, model-00001-of-00002.safetensors
+3, model.layers.0.mlp.gate_proj.weight, torch.Size([8192, 2560]), layer 0, model-00001-of-00002.safetensors
+3, model.layers.0.mlp.up_proj.weight, torch.Size([8192, 2560]), layer 0, model-00001-of-00002.safetensors
+3, model.layers.0.post_attention_layernorm.weight, torch.Size([2560]), layer 0, model-00001-of-00002.safetensors
+3, model.layers.0.self_attn.k_proj.weight, torch.Size([512, 2560]), layer 0, model-00001-of-00002.safetensors
+1, model.layers.0.self_attn.o_proj.weight, torch.Size([2560, 2560]), layer 0, model-00001-of-00002.safetensors
+3, model.layers.0.self_attn.q_proj.weight, torch.Size([2560, 2560]), layer 0, model-00001-of-00002.safetensors
+3, model.layers.0.self_attn.v_proj.weight, torch.Size([512, 2560]), layer 0, model-00001-of-00002.safetensors
+3, model.layers.1.input_layernorm.weight, torch.Size([2560]), layer 1, model-00001-of-00002.safetensors
+2, model.layers.1.mlp.down_proj.weight, torch.Size([2560, 8192]), layer 1, model-00001-of-00002.safetensors
+3, model.layers.1.mlp.gate_proj.weight, torch.Size([8192, 2560]), layer 1, model-00001-of-00002.safetensors
+3, model.layers.1.mlp.up_proj.weight, torch.Size([8192, 2560]), layer 1, model-00001-of-00002.safetensors
+3, model.layers.1.post_attention_layernorm.weight, torch.Size([2560]), layer 1, model-00001-of-00002.safetensors
+3, model.layers.1.self_attn.k_proj.weight, torch.Size([512, 2560]), layer 1, model-00001-of-00002.safetensors
+1, model.layers.1.self_attn.o_proj.weight, torch.Size([2560, 2560]), layer 1, model-00001-of-00002.safetensors
+3, model.layers.1.self_attn.q_proj.weight, torch.Size([2560, 2560]), layer 1, model-00001-of-00002.safetensors
+3, model.layers.1.self_attn.v_proj.weight, torch.Size([512, 2560]), layer 1, model-00001-of-00002.safetensors
+3, model.layers.10.input_layernorm.weight, torch.Size([2560]), layer 10, model-00001-of-00002.safetensors
+2, model.layers.10.mlp.down_proj.weight, torch.Size([2560, 8192]), layer 10, model-00001-of-00002.safetensors
+3, model.layers.10.mlp.gate_proj.weight, torch.Size([8192, 2560]), layer 10, model-00001-of-00002.safetensors
+3, model.layers.10.mlp.up_proj.weight, torch.Size([8192, 2560]), layer 10, model-00001-of-00002.safetensors
+3, model.layers.10.post_attention_layernorm.weight, torch.Size([2560]), layer 10, model-00001-of-00002.safetensors
+3, model.layers.10.self_attn.k_proj.weight, torch.Size([512, 2560]), layer 10, model-00001-of-00002.safetensors
+1, model.layers.10.self_attn.o_proj.weight, torch.Size([2560, 2560]), layer 10, model-00001-of-00002.safetensors
+3, model.layers.10.self_attn.q_proj.weight, torch.Size([2560, 2560]), layer 10, model-00001-of-00002.safetensors
+3, model.layers.10.self_attn.v_proj.weight, torch.Size([512, 2560]), layer 10, model-00001-of-00002.safetensors
+3, model.layers.11.input_layernorm.weight, torch.Size([2560]), layer 11, model-00001-of-00002.safetensors
+2, model.layers.11.mlp.down_proj.weight, torch.Size([2560, 8192]), layer 11, model-00001-of-00002.safetensors
+3, model.layers.11.mlp.gate_proj.weight, torch.Size([8192, 2560]), layer 11, model-00001-of-00002.safetensors
+3, model.layers.11.mlp.up_proj.weight, torch.Size([8192, 2560]), layer 11, model-00001-of-00002.safetensors
+3, model.layers.11.post_attention_layernorm.weight, torch.Size([2560]), layer 11, model-00001-of-00002.safetensors
+3, model.layers.11.self_attn.k_proj.weight, torch.Size([512, 2560]), layer 11, model-00001-of-00002.safetensors
+1, model.layers.11.self_attn.o_proj.weight, torch.Size([2560, 2560]), layer 11, model-00001-of-00002.safetensors
+3, model.layers.11.self_attn.q_proj.weight, torch.Size([2560, 2560]), layer 11, model-00001-of-00002.safetensors
+3, model.layers.11.self_attn.v_proj.weight, torch.Size([512, 2560]), layer 11, model-00001-of-00002.safetensors
+3, model.layers.12.input_layernorm.weight, torch.Size([2560]), layer 12, model-00001-of-00002.safetensors
+2, model.layers.12.mlp.down_proj.weight, torch.Size([2560, 8192]), layer 12, model-00001-of-00002.safetensors
+3, model.layers.12.mlp.gate_proj.weight, torch.Size([8192, 2560]), layer 12, model-00001-of-00002.safetensors
+3, model.layers.12.mlp.up_proj.weight, torch.Size([8192, 2560]), layer 12, model-00001-of-00002.safetensors
+3, model.layers.12.post_attention_layernorm.weight, torch.Size([2560]), layer 12, model-00001-of-00002.safetensors
+3, model.layers.12.self_attn.k_proj.weight, torch.Size([512, 2560]), layer 12, model-00001-of-00002.safetensors
+1, model.layers.12.self_attn.o_proj.weight, torch.Size([2560, 2560]), layer 12, model-00001-of-00002.safetensors
+3, model.layers.12.self_attn.q_proj.weight, torch.Size([2560, 2560]), layer 12, model-00001-of-00002.safetensors
+3, model.layers.12.self_attn.v_proj.weight, torch.Size([512, 2560]), layer 12, model-00001-of-00002.safetensors
+3, model.layers.13.input_layernorm.weight, torch.Size([2560]), layer 13, model-00001-of-00002.safetensors
+2, model.layers.13.mlp.down_proj.weight, torch.Size([2560, 8192]), layer 13, model-00001-of-00002.safetensors
+3, model.layers.13.mlp.gate_proj.weight, torch.Size([8192, 2560]), layer 13, model-00001-of-00002.safetensors
+3, model.layers.13.mlp.up_proj.weight, torch.Size([8192, 2560]), layer 13, model-00001-of-00002.safetensors
+3, model.layers.13.post_attention_layernorm.weight, torch.Size([2560]), layer 13, model-00001-of-00002.safetensors
+3, model.layers.13.self_attn.k_proj.weight, torch.Size([512, 2560]), layer 13, model-00001-of-00002.safetensors
+1, model.layers.13.self_attn.o_proj.weight, torch.Size([2560, 2560]), layer 13, model-00001-of-00002.safetensors
+3, model.layers.13.self_attn.q_proj.weight, torch.Size([2560, 2560]), layer 13, model-00001-of-00002.safetensors
+3, model.layers.13.self_attn.v_proj.weight, torch.Size([512, 2560]), layer 13, model-00001-of-00002.safetensors
+3, model.layers.14.input_layernorm.weight, torch.Size([2560]), layer 14, model-00001-of-00002.safetensors
+2, model.layers.14.mlp.down_proj.weight, torch.Size([2560, 8192]), layer 14, model-00001-of-00002.safetensors
+3, model.layers.14.mlp.gate_proj.weight, torch.Size([8192, 2560]), layer 14, model-00001-of-00002.safetensors
+3, model.layers.14.mlp.up_proj.weight, torch.Size([8192, 2560]), layer 14, model-00001-of-00002.safetensors
+3, model.layers.14.post_attention_layernorm.weight, torch.Size([2560]), layer 14, model-00001-of-00002.safetensors
+3, model.layers.14.self_attn.k_proj.weight, torch.Size([512, 2560]), layer 14, model-00001-of-00002.safetensors
+1, model.layers.14.self_attn.o_proj.weight, torch.Size([2560, 2560]), layer 14, model-00001-of-00002.safetensors
+3, model.layers.14.self_attn.q_proj.weight, torch.Size([2560, 2560]), layer 14, model-00001-of-00002.safetensors
+3, model.layers.14.self_attn.v_proj.weight, torch.Size([512, 2560]), layer 14, model-00001-of-00002.safetensors
+3, model.layers.15.input_layernorm.weight, torch.Size([2560]), layer 15, model-00001-of-00002.safetensors
+2, model.layers.15.mlp.down_proj.weight, torch.Size([2560, 8192]), layer 15, model-00001-of-00002.safetensors
+3, model.layers.15.mlp.gate_proj.weight, torch.Size([8192, 2560]), layer 15, model-00001-of-00002.safetensors
+3, model.layers.15.mlp.up_proj.weight, torch.Size([8192, 2560]), layer 15, model-00001-of-00002.safetensors
+3, model.layers.15.post_attention_layernorm.weight, torch.Size([2560]), layer 15, model-00001-of-00002.safetensors
+3, model.layers.15.self_attn.k_proj.weight, torch.Size([512, 2560]), layer 15, model-00001-of-00002.safetensors
+1, model.layers.15.self_attn.o_proj.weight, torch.Size([2560, 2560]), layer 15, model-00001-of-00002.safetensors
+3, model.layers.15.self_attn.q_proj.weight, torch.Size([2560, 2560]), layer 15, model-00001-of-00002.safetensors
+3, model.layers.15.self_attn.v_proj.weight, torch.Size([512, 2560]), layer 15, model-00001-of-00002.safetensors
+3, model.layers.16.input_layernorm.weight, torch.Size([2560]), layer 16, model-00001-of-00002.safetensors
+2, model.layers.16.mlp.down_proj.weight, torch.Size([2560, 8192]), layer 16, model-00001-of-00002.safetensors
+3, model.layers.16.mlp.gate_proj.weight, torch.Size([8192, 2560]), layer 16, model-00001-of-00002.safetensors
+3, model.layers.16.mlp.up_proj.weight, torch.Size([8192, 2560]), layer 16, model-00001-of-00002.safetensors
+3, model.layers.16.post_attention_layernorm.weight, torch.Size([2560]), layer 16, model-00001-of-00002.safetensors
+3, model.layers.16.self_attn.k_proj.weight, torch.Size([512, 2560]), layer 16, model-00001-of-00002.safetensors
+1, model.layers.16.self_attn.o_proj.weight, torch.Size([2560, 2560]), layer 16, model-00001-of-00002.safetensors
+3, model.layers.16.self_attn.q_proj.weight, torch.Size([2560, 2560]), layer 16, model-00001-of-00002.safetensors
+3, model.layers.16.self_attn.v_proj.weight, torch.Size([512, 2560]), layer 16, model-00001-of-00002.safetensors
+3, model.layers.17.input_layernorm.weight, torch.Size([2560]), layer 17, model-00001-of-00002.safetensors
+2, model.layers.17.mlp.down_proj.weight, torch.Size([2560, 8192]), layer 17, model-00001-of-00002.safetensors
+3, model.layers.17.mlp.gate_proj.weight, torch.Size([8192, 2560]), layer 17, model-00001-of-00002.safetensors
+3, model.layers.17.mlp.up_proj.weight, torch.Size([8192, 2560]), layer 17, model-00001-of-00002.safetensors
+3, model.layers.17.post_attention_layernorm.weight, torch.Size([2560]), layer 17, model-00001-of-00002.safetensors
+3, model.layers.17.self_attn.k_proj.weight, torch.Size([512, 2560]), layer 17, model-00001-of-00002.safetensors
+1, model.layers.17.self_attn.o_proj.weight, torch.Size([2560, 2560]), layer 17, model-00001-of-00002.safetensors
+3, model.layers.17.self_attn.q_proj.weight, torch.Size([2560, 2560]), layer 17, model-00001-of-00002.safetensors
+3, model.layers.17.self_attn.v_proj.weight, torch.Size([512, 2560]), layer 17, model-00001-of-00002.safetensors
+3, model.layers.18.input_layernorm.weight, torch.Size([2560]), layer 18, model-00001-of-00002.safetensors
+2, model.layers.18.mlp.down_proj.weight, torch.Size([2560, 8192]), layer 18, model-00001-of-00002.safetensors
+3, model.layers.18.mlp.gate_proj.weight, torch.Size([8192, 2560]), layer 18, model-00001-of-00002.safetensors
+3, model.layers.18.mlp.up_proj.weight, torch.Size([8192, 2560]), layer 18, model-00001-of-00002.safetensors
+3, model.layers.18.post_attention_layernorm.weight, torch.Size([2560]), layer 18, model-00001-of-00002.safetensors
+3, model.layers.18.self_attn.k_proj.weight, torch.Size([512, 2560]), layer 18, model-00001-of-00002.safetensors
+1, model.layers.18.self_attn.o_proj.weight, torch.Size([2560, 2560]), layer 18, model-00001-of-00002.safetensors
+3, model.layers.18.self_attn.q_proj.weight, torch.Size([2560, 2560]), layer 18, model-00001-of-00002.safetensors
+3, model.layers.18.self_attn.v_proj.weight, torch.Size([512, 2560]), layer 18, model-00001-of-00002.safetensors
+3, model.layers.19.input_layernorm.weight, torch.Size([2560]), layer 19, model-00001-of-00002.safetensors
+2, model.layers.19.mlp.down_proj.weight, torch.Size([2560, 8192]), layer 19, model-00001-of-00002.safetensors
+3, model.layers.19.mlp.gate_proj.weight, torch.Size([8192, 2560]), layer 19, model-00001-of-00002.safetensors
+3, model.layers.19.mlp.up_proj.weight, torch.Size([8192, 2560]), layer 19, model-00001-of-00002.safetensors
+3, model.layers.19.post_attention_layernorm.weight, torch.Size([2560]), layer 19, model-00001-of-00002.safetensors
+3, model.layers.19.self_attn.k_proj.weight, torch.Size([512, 2560]), layer 19, model-00001-of-00002.safetensors
+1, model.layers.19.self_attn.o_proj.weight, torch.Size([2560, 2560]), layer 19, model-00001-of-00002.safetensors
+3, model.layers.19.self_attn.q_proj.weight, torch.Size([2560, 2560]), layer 19, model-00001-of-00002.safetensors
+3, model.layers.19.self_attn.v_proj.weight, torch.Size([512, 2560]), layer 19, model-00001-of-00002.safetensors
+3, model.layers.2.input_layernorm.weight, torch.Size([2560]), layer 2, model-00001-of-00002.safetensors
+2, model.layers.2.mlp.down_proj.weight, torch.Size([2560, 8192]), layer 2, model-00001-of-00002.safetensors
+3, model.layers.2.mlp.gate_proj.weight, torch.Size([8192, 2560]), layer 2, model-00001-of-00002.safetensors
+3, model.layers.2.mlp.up_proj.weight, torch.Size([8192, 2560]), layer 2, model-00001-of-00002.safetensors
+3, model.layers.2.post_attention_layernorm.weight, torch.Size([2560]), layer 2, model-00001-of-00002.safetensors
+3, model.layers.2.self_attn.k_proj.weight, torch.Size([512, 2560]), layer 2, model-00001-of-00002.safetensors
+1, model.layers.2.self_attn.o_proj.weight, torch.Size([2560, 2560]), layer 2, model-00001-of-00002.safetensors
+3, model.layers.2.self_attn.q_proj.weight, torch.Size([2560, 2560]), layer 2, model-00001-of-00002.safetensors
+3, model.layers.2.self_attn.v_proj.weight, torch.Size([512, 2560]), layer 2, model-00001-of-00002.safetensors
+3, model.layers.20.input_layernorm.weight, torch.Size([2560]), layer 20, model-00001-of-00002.safetensors
+2, model.layers.20.mlp.down_proj.weight, torch.Size([2560, 8192]), layer 20, model-00001-of-00002.safetensors
+3, model.layers.20.mlp.gate_proj.weight, torch.Size([8192, 2560]), layer 20, model-00001-of-00002.safetensors
+3, model.layers.20.mlp.up_proj.weight, torch.Size([8192, 2560]), layer 20, model-00001-of-00002.safetensors
+3, model.layers.20.post_attention_layernorm.weight, torch.Size([2560]), layer 20, model-00001-of-00002.safetensors
+3, model.layers.20.self_attn.k_proj.weight, torch.Size([512, 2560]), layer 20, model-00001-of-00002.safetensors
+1, model.layers.20.self_attn.o_proj.weight, torch.Size([2560, 2560]), layer 20, model-00001-of-00002.safetensors
+3, model.layers.20.self_attn.q_proj.weight, torch.Size([2560, 2560]), layer 20, model-00001-of-00002.safetensors
+3, model.layers.20.self_attn.v_proj.weight, torch.Size([512, 2560]), layer 20, model-00001-of-00002.safetensors
+3, model.layers.21.input_layernorm.weight, torch.Size([2560]), layer 21, model-00001-of-00002.safetensors
+2, model.layers.21.mlp.down_proj.weight, torch.Size([2560, 8192]), layer 21, model-00001-of-00002.safetensors
+3, model.layers.21.mlp.gate_proj.weight, torch.Size([8192, 2560]), layer 21, model-00001-of-00002.safetensors
+3, model.layers.21.mlp.up_proj.weight, torch.Size([8192, 2560]), layer 21, model-00001-of-00002.safetensors
+3, model.layers.21.post_attention_layernorm.weight, torch.Size([2560]), layer 21, model-00001-of-00002.safetensors
+3, model.layers.21.self_attn.k_proj.weight, torch.Size([512, 2560]), layer 21, model-00001-of-00002.safetensors
+1, model.layers.21.self_attn.o_proj.weight, torch.Size([2560, 2560]), layer 21, model-00001-of-00002.safetensors
+3, model.layers.21.self_attn.q_proj.weight, torch.Size([2560, 2560]), layer 21, model-00001-of-00002.safetensors
+3, model.layers.21.self_attn.v_proj.weight, torch.Size([512, 2560]), layer 21, model-00001-of-00002.safetensors
+3, model.layers.22.input_layernorm.weight, torch.Size([2560]), layer 22, model-00001-of-00002.safetensors
+2, model.layers.22.mlp.down_proj.weight, torch.Size([2560, 8192]), layer 22, model-00001-of-00002.safetensors
+3, model.layers.22.mlp.gate_proj.weight, torch.Size([8192, 2560]), layer 22, model-00001-of-00002.safetensors
+3, model.layers.22.mlp.up_proj.weight, torch.Size([8192, 2560]), layer 22, model-00001-of-00002.safetensors
+3, model.layers.22.post_attention_layernorm.weight, torch.Size([2560]), layer 22, model-00001-of-00002.safetensors
+3, model.layers.22.self_attn.k_proj.weight, torch.Size([512, 2560]), layer 22, model-00001-of-00002.safetensors
+1, model.layers.22.self_attn.o_proj.weight, torch.Size([2560, 2560]), layer 22, model-00001-of-00002.safetensors
+3, model.layers.22.self_attn.q_proj.weight, torch.Size([2560, 2560]), layer 22, model-00001-of-00002.safetensors
+3, model.layers.22.self_attn.v_proj.weight, torch.Size([512, 2560]), layer 22, model-00001-of-00002.safetensors
+3, model.layers.23.input_layernorm.weight, torch.Size([2560]), layer 23, model-00001-of-00002.safetensors
+2, model.layers.23.mlp.down_proj.weight, torch.Size([2560, 8192]), layer 23, model-00001-of-00002.safetensors
+3, model.layers.23.mlp.gate_proj.weight, torch.Size([8192, 2560]), layer 23, model-00001-of-00002.safetensors
+3, model.layers.23.mlp.up_proj.weight, torch.Size([8192, 2560]), layer 23, model-00001-of-00002.safetensors
+3, model.layers.23.post_attention_layernorm.weight, torch.Size([2560]), layer 23, model-00001-of-00002.safetensors
+3, model.layers.23.self_attn.k_proj.weight, torch.Size([512, 2560]), layer 23, model-00001-of-00002.safetensors
+1, model.layers.23.self_attn.o_proj.weight, torch.Size([2560, 2560]), layer 23, model-00001-of-00002.safetensors
+3, model.layers.23.self_attn.q_proj.weight, torch.Size([2560, 2560]), layer 23, model-00001-of-00002.safetensors
+3, model.layers.23.self_attn.v_proj.weight, torch.Size([512, 2560]), layer 23, model-00001-of-00002.safetensors
+3, model.layers.24.input_layernorm.weight, torch.Size([2560]), layer 24, model-00001-of-00002.safetensors
+2, model.layers.24.mlp.down_proj.weight, torch.Size([2560, 8192]), layer 24, model-00001-of-00002.safetensors
+3, model.layers.24.mlp.gate_proj.weight, torch.Size([8192, 2560]), layer 24, model-00001-of-00002.safetensors
+3, model.layers.24.mlp.up_proj.weight, torch.Size([8192, 2560]), layer 24, model-00001-of-00002.safetensors
+3, model.layers.24.post_attention_layernorm.weight, torch.Size([2560]), layer 24, model-00001-of-00002.safetensors
+3, model.layers.24.self_attn.k_proj.weight, torch.Size([512, 2560]), layer 24, model-00001-of-00002.safetensors
+1, model.layers.24.self_attn.o_proj.weight, torch.Size([2560, 2560]), layer 24, model-00001-of-00002.safetensors
+3, model.layers.24.self_attn.q_proj.weight, torch.Size([2560, 2560]), layer 24, model-00001-of-00002.safetensors
+3, model.layers.24.self_attn.v_proj.weight, torch.Size([512, 2560]), layer 24, model-00001-of-00002.safetensors
+3, model.layers.25.input_layernorm.weight, torch.Size([2560]), layer 25, model-00001-of-00002.safetensors
+2, model.layers.25.mlp.down_proj.weight, torch.Size([2560, 8192]), layer 25, model-00001-of-00002.safetensors
+3, model.layers.25.mlp.gate_proj.weight, torch.Size([8192, 2560]), layer 25, model-00001-of-00002.safetensors
+3, model.layers.25.mlp.up_proj.weight, torch.Size([8192, 2560]), layer 25, model-00001-of-00002.safetensors
+3, model.layers.25.post_attention_layernorm.weight, torch.Size([2560]), layer 25, model-00001-of-00002.safetensors
+3, model.layers.25.self_attn.k_proj.weight, torch.Size([512, 2560]), layer 25, model-00001-of-00002.safetensors
+1, model.layers.25.self_attn.o_proj.weight, torch.Size([2560, 2560]), layer 25, model-00001-of-00002.safetensors
+3, model.layers.25.self_attn.q_proj.weight, torch.Size([2560, 2560]), layer 25, model-00001-of-00002.safetensors
+3, model.layers.25.self_attn.v_proj.weight, torch.Size([512, 2560]), layer 25, model-00001-of-00002.safetensors
+3, model.layers.26.input_layernorm.weight, torch.Size([2560]), layer 26, model-00001-of-00002.safetensors
+2, model.layers.26.mlp.down_proj.weight, torch.Size([2560, 8192]), layer 26, model-00001-of-00002.safetensors
+3, model.layers.26.mlp.gate_proj.weight, torch.Size([8192, 2560]), layer 26, model-00001-of-00002.safetensors
+3, model.layers.26.mlp.up_proj.weight, torch.Size([8192, 2560]), layer 26, model-00001-of-00002.safetensors
+3, model.layers.26.post_attention_layernorm.weight, torch.Size([2560]), layer 26, model-00001-of-00002.safetensors
+3, model.layers.26.self_attn.k_proj.weight, torch.Size([512, 2560]), layer 26, model-00001-of-00002.safetensors
+1, model.layers.26.self_attn.o_proj.weight, torch.Size([2560, 2560]), layer 26, model-00001-of-00002.safetensors
+3, model.layers.26.self_attn.q_proj.weight, torch.Size([2560, 2560]), layer 26, model-00001-of-00002.safetensors
+3, model.layers.26.self_attn.v_proj.weight, torch.Size([512, 2560]), layer 26, model-00001-of-00002.safetensors
+3, model.layers.27.input_layernorm.weight, torch.Size([2560]), layer 27, model-00001-of-00002.safetensors
+2, model.layers.27.mlp.down_proj.weight, torch.Size([2560, 8192]), layer 27, model-00001-of-00002.safetensors
+3, model.layers.27.mlp.gate_proj.weight, torch.Size([8192, 2560]), layer 27, model-00001-of-00002.safetensors
+3, model.layers.27.mlp.up_proj.weight, torch.Size([8192, 2560]), layer 27, model-00001-of-00002.safetensors
+3, model.layers.27.post_attention_layernorm.weight, torch.Size([2560]), layer 27, model-00001-of-00002.safetensors
+3, model.layers.27.self_attn.k_proj.weight, torch.Size([512, 2560]), layer 27, model-00001-of-00002.safetensors
+1, model.layers.27.self_attn.o_proj.weight, torch.Size([2560, 2560]), layer 27, model-00001-of-00002.safetensors
+3, model.layers.27.self_attn.q_proj.weight, torch.Size([2560, 2560]), layer 27, model-00001-of-00002.safetensors
+3, model.layers.27.self_attn.v_proj.weight, torch.Size([512, 2560]), layer 27, model-00001-of-00002.safetensors
+3, model.layers.28.mlp.gate_proj.weight, torch.Size([8192, 2560]), layer 28, model-00001-of-00002.safetensors
+3, model.layers.28.self_attn.k_proj.weight, torch.Size([512, 2560]), layer 28, model-00001-of-00002.safetensors
+1, model.layers.28.self_attn.o_proj.weight, torch.Size([2560, 2560]), layer 28, model-00001-of-00002.safetensors
+3, model.layers.28.self_attn.q_proj.weight, torch.Size([2560, 2560]), layer 28, model-00001-of-00002.safetensors
+3, model.layers.28.self_attn.v_proj.weight, torch.Size([512, 2560]), layer 28, model-00001-of-00002.safetensors
+3, model.layers.3.input_layernorm.weight, torch.Size([2560]), layer 3, model-00001-of-00002.safetensors
+2, model.layers.3.mlp.down_proj.weight, torch.Size([2560, 8192]), layer 3, model-00001-of-00002.safetensors
+3, model.layers.3.mlp.gate_proj.weight, torch.Size([8192, 2560]), layer 3, model-00001-of-00002.safetensors
+3, model.layers.3.mlp.up_proj.weight, torch.Size([8192, 2560]), layer 3, model-00001-of-00002.safetensors
+3, model.layers.3.post_attention_layernorm.weight, torch.Size([2560]), layer 3, model-00001-of-00002.safetensors
+3, model.layers.3.self_attn.k_proj.weight, torch.Size([512, 2560]), layer 3, model-00001-of-00002.safetensors
+1, model.layers.3.self_attn.o_proj.weight, torch.Size([2560, 2560]), layer 3, model-00001-of-00002.safetensors
+3, model.layers.3.self_attn.q_proj.weight, torch.Size([2560, 2560]), layer 3, model-00001-of-00002.safetensors
+3, model.layers.3.self_attn.v_proj.weight, torch.Size([512, 2560]), layer 3, model-00001-of-00002.safetensors
+3, model.layers.4.input_layernorm.weight, torch.Size([2560]), layer 4, model-00001-of-00002.safetensors
+2, model.layers.4.mlp.down_proj.weight, torch.Size([2560, 8192]), layer 4, model-00001-of-00002.safetensors
+3, model.layers.4.mlp.gate_proj.weight, torch.Size([8192, 2560]), layer 4, model-00001-of-00002.safetensors
+3, model.layers.4.mlp.up_proj.weight, torch.Size([8192, 2560]), layer 4, model-00001-of-00002.safetensors
+3, model.layers.4.post_attention_layernorm.weight, torch.Size([2560]), layer 4, model-00001-of-00002.safetensors
+3, model.layers.4.self_attn.k_proj.weight, torch.Size([512, 2560]), layer 4, model-00001-of-00002.safetensors
+1, model.layers.4.self_attn.o_proj.weight, torch.Size([2560, 2560]), layer 4, model-00001-of-00002.safetensors
+3, model.layers.4.self_attn.q_proj.weight, torch.Size([2560, 2560]), layer 4, model-00001-of-00002.safetensors
+3, model.layers.4.self_attn.v_proj.weight, torch.Size([512, 2560]), layer 4, model-00001-of-00002.safetensors
+3, model.layers.5.input_layernorm.weight, torch.Size([2560]), layer 5, model-00001-of-00002.safetensors
+2, model.layers.5.mlp.down_proj.weight, torch.Size([2560, 8192]), layer 5, model-00001-of-00002.safetensors
+3, model.layers.5.mlp.gate_proj.weight, torch.Size([8192, 2560]), layer 5, model-00001-of-00002.safetensors
+3, model.layers.5.mlp.up_proj.weight, torch.Size([8192, 2560]), layer 5, model-00001-of-00002.safetensors
+3, model.layers.5.post_attention_layernorm.weight, torch.Size([2560]), layer 5, model-00001-of-00002.safetensors
+3, model.layers.5.self_attn.k_proj.weight, torch.Size([512, 2560]), layer 5, model-00001-of-00002.safetensors
+1, model.layers.5.self_attn.o_proj.weight, torch.Size([2560, 2560]), layer 5, model-00001-of-00002.safetensors
+3, model.layers.5.self_attn.q_proj.weight, torch.Size([2560, 2560]), layer 5, model-00001-of-00002.safetensors
+3, model.layers.5.self_attn.v_proj.weight, torch.Size([512, 2560]), layer 5, model-00001-of-00002.safetensors
+3, model.layers.6.input_layernorm.weight, torch.Size([2560]), layer 6, model-00001-of-00002.safetensors
+2, model.layers.6.mlp.down_proj.weight, torch.Size([2560, 8192]), layer 6, model-00001-of-00002.safetensors
+3, model.layers.6.mlp.gate_proj.weight, torch.Size([8192, 2560]), layer 6, model-00001-of-00002.safetensors
+3, model.layers.6.mlp.up_proj.weight, torch.Size([8192, 2560]), layer 6, model-00001-of-00002.safetensors
+3, model.layers.6.post_attention_layernorm.weight, torch.Size([2560]), layer 6, model-00001-of-00002.safetensors
+3, model.layers.6.self_attn.k_proj.weight, torch.Size([512, 2560]), layer 6, model-00001-of-00002.safetensors
+1, model.layers.6.self_attn.o_proj.weight, torch.Size([2560, 2560]), layer 6, model-00001-of-00002.safetensors
+3, model.layers.6.self_attn.q_proj.weight, torch.Size([2560, 2560]), layer 6, model-00001-of-00002.safetensors
+3, model.layers.6.self_attn.v_proj.weight, torch.Size([512, 2560]), layer 6, model-00001-of-00002.safetensors
+3, model.layers.7.input_layernorm.weight, torch.Size([2560]), layer 7, model-00001-of-00002.safetensors
+2, model.layers.7.mlp.down_proj.weight, torch.Size([2560, 8192]), layer 7, model-00001-of-00002.safetensors
+3, model.layers.7.mlp.gate_proj.weight, torch.Size([8192, 2560]), layer 7, model-00001-of-00002.safetensors
+3, model.layers.7.mlp.up_proj.weight, torch.Size([8192, 2560]), layer 7, model-00001-of-00002.safetensors
+3, model.layers.7.post_attention_layernorm.weight, torch.Size([2560]), layer 7, model-00001-of-00002.safetensors
+3, model.layers.7.self_attn.k_proj.weight, torch.Size([512, 2560]), layer 7, model-00001-of-00002.safetensors
+1, model.layers.7.self_attn.o_proj.weight, torch.Size([2560, 2560]), layer 7, model-00001-of-00002.safetensors
+3, model.layers.7.self_attn.q_proj.weight, torch.Size([2560, 2560]), layer 7, model-00001-of-00002.safetensors
+3, model.layers.7.self_attn.v_proj.weight, torch.Size([512, 2560]), layer 7, model-00001-of-00002.safetensors
+3, model.layers.8.input_layernorm.weight, torch.Size([2560]), layer 8, model-00001-of-00002.safetensors
+2, model.layers.8.mlp.down_proj.weight, torch.Size([2560, 8192]), layer 8, model-00001-of-00002.safetensors
+3, model.layers.8.mlp.gate_proj.weight, torch.Size([8192, 2560]), layer 8, model-00001-of-00002.safetensors
+3, model.layers.8.mlp.up_proj.weight, torch.Size([8192, 2560]), layer 8, model-00001-of-00002.safetensors
+3, model.layers.8.post_attention_layernorm.weight, torch.Size([2560]), layer 8, model-00001-of-00002.safetensors
+3, model.layers.8.self_attn.k_proj.weight, torch.Size([512, 2560]), layer 8, model-00001-of-00002.safetensors
+1, model.layers.8.self_attn.o_proj.weight, torch.Size([2560, 2560]), layer 8, model-00001-of-00002.safetensors
+3, model.layers.8.self_attn.q_proj.weight, torch.Size([2560, 2560]), layer 8, model-00001-of-00002.safetensors
+3, model.layers.8.self_attn.v_proj.weight, torch.Size([512, 2560]), layer 8, model-00001-of-00002.safetensors
+3, model.layers.9.input_layernorm.weight, torch.Size([2560]), layer 9, model-00001-of-00002.safetensors
+2, model.layers.9.mlp.down_proj.weight, torch.Size([2560, 8192]), layer 9, model-00001-of-00002.safetensors
+3, model.layers.9.mlp.gate_proj.weight, torch.Size([8192, 2560]), layer 9, model-00001-of-00002.safetensors
+3, model.layers.9.mlp.up_proj.weight, torch.Size([8192, 2560]), layer 9, model-00001-of-00002.safetensors
+3, model.layers.9.post_attention_layernorm.weight, torch.Size([2560]), layer 9, model-00001-of-00002.safetensors
+3, model.layers.9.self_attn.k_proj.weight, torch.Size([512, 2560]), layer 9, model-00001-of-00002.safetensors
+1, model.layers.9.self_attn.o_proj.weight, torch.Size([2560, 2560]), layer 9, model-00001-of-00002.safetensors
+3, model.layers.9.self_attn.q_proj.weight, torch.Size([2560, 2560]), layer 9, model-00001-of-00002.safetensors
+3, model.layers.9.self_attn.v_proj.weight, torch.Size([512, 2560]), layer 9, model-00001-of-00002.safetensors
+3, model.layers.28.input_layernorm.weight, torch.Size([2560]), layer 28, model-00002-of-00002.safetensors
+2, model.layers.28.mlp.down_proj.weight, torch.Size([2560, 8192]), layer 28, model-00002-of-00002.safetensors
+3, model.layers.28.mlp.up_proj.weight, torch.Size([8192, 2560]), layer 28, model-00002-of-00002.safetensors
+3, model.layers.28.post_attention_layernorm.weight, torch.Size([2560]), layer 28, model-00002-of-00002.safetensors
+3, model.layers.29.input_layernorm.weight, torch.Size([2560]), layer 29, model-00002-of-00002.safetensors
+2, model.layers.29.mlp.down_proj.weight, torch.Size([2560, 8192]), layer 29, model-00002-of-00002.safetensors
+3, model.layers.29.mlp.gate_proj.weight, torch.Size([8192, 2560]), layer 29, model-00002-of-00002.safetensors
+3, model.layers.29.mlp.up_proj.weight, torch.Size([8192, 2560]), layer 29, model-00002-of-00002.safetensors
+3, model.layers.29.post_attention_layernorm.weight, torch.Size([2560]), layer 29, model-00002-of-00002.safetensors
+3, model.layers.29.self_attn.k_proj.weight, torch.Size([512, 2560]), layer 29, model-00002-of-00002.safetensors
+1, model.layers.29.self_attn.o_proj.weight, torch.Size([2560, 2560]), layer 29, model-00002-of-00002.safetensors
+3, model.layers.29.self_attn.q_proj.weight, torch.Size([2560, 2560]), layer 29, model-00002-of-00002.safetensors
+3, model.layers.29.self_attn.v_proj.weight, torch.Size([512, 2560]), layer 29, model-00002-of-00002.safetensors
+3, model.layers.30.input_layernorm.weight, torch.Size([2560]), layer 30, model-00002-of-00002.safetensors
+2, model.layers.30.mlp.down_proj.weight, torch.Size([2560, 8192]), layer 30, model-00002-of-00002.safetensors
+3, model.layers.30.mlp.gate_proj.weight, torch.Size([8192, 2560]), layer 30, model-00002-of-00002.safetensors
+3, model.layers.30.mlp.up_proj.weight, torch.Size([8192, 2560]), layer 30, model-00002-of-00002.safetensors
+3, model.layers.30.post_attention_layernorm.weight, torch.Size([2560]), layer 30, model-00002-of-00002.safetensors
+3, model.layers.30.self_attn.k_proj.weight, torch.Size([512, 2560]), layer 30, model-00002-of-00002.safetensors
+1, model.layers.30.self_attn.o_proj.weight, torch.Size([2560, 2560]), layer 30, model-00002-of-00002.safetensors
+3, model.layers.30.self_attn.q_proj.weight, torch.Size([2560, 2560]), layer 30, model-00002-of-00002.safetensors
+3, model.layers.30.self_attn.v_proj.weight, torch.Size([512, 2560]), layer 30, model-00002-of-00002.safetensors
+3, model.layers.31.input_layernorm.weight, torch.Size([2560]), layer 31, model-00002-of-00002.safetensors
+2, model.layers.31.mlp.down_proj.weight, torch.Size([2560, 8192]), layer 31, model-00002-of-00002.safetensors
+3, model.layers.31.mlp.gate_proj.weight, torch.Size([8192, 2560]), layer 31, model-00002-of-00002.safetensors
+3, model.layers.31.mlp.up_proj.weight, torch.Size([8192, 2560]), layer 31, model-00002-of-00002.safetensors
+3, model.layers.31.post_attention_layernorm.weight, torch.Size([2560]), layer 31, model-00002-of-00002.safetensors
+3, model.layers.31.self_attn.k_proj.weight, torch.Size([512, 2560]), layer 31, model-00002-of-00002.safetensors
+1, model.layers.31.self_attn.o_proj.weight, torch.Size([2560, 2560]), layer 31, model-00002-of-00002.safetensors
+3, model.layers.31.self_attn.q_proj.weight, torch.Size([2560, 2560]), layer 31, model-00002-of-00002.safetensors
+3, model.layers.31.self_attn.v_proj.weight, torch.Size([512, 2560]), layer 31, model-00002-of-00002.safetensors
+3, model.layers.32.input_layernorm.weight, torch.Size([2560]), layer 32, model-00002-of-00002.safetensors
+2, model.layers.32.mlp.down_proj.weight, torch.Size([2560, 8192]), layer 32, model-00002-of-00002.safetensors
+3, model.layers.32.mlp.gate_proj.weight, torch.Size([8192, 2560]), layer 32, model-00002-of-00002.safetensors
+3, model.layers.32.mlp.up_proj.weight, torch.Size([8192, 2560]), layer 32, model-00002-of-00002.safetensors
+3, model.layers.32.post_attention_layernorm.weight, torch.Size([2560]), layer 32, model-00002-of-00002.safetensors
+3, model.layers.32.self_attn.k_proj.weight, torch.Size([512, 2560]), layer 32, model-00002-of-00002.safetensors
+1, model.layers.32.self_attn.o_proj.weight, torch.Size([2560, 2560]), layer 32, model-00002-of-00002.safetensors
+3, model.layers.32.self_attn.q_proj.weight, torch.Size([2560, 2560]), layer 32, model-00002-of-00002.safetensors
+3, model.layers.32.self_attn.v_proj.weight, torch.Size([512, 2560]), layer 32, model-00002-of-00002.safetensors
+3, model.layers.33.input_layernorm.weight, torch.Size([2560]), layer 33, model-00002-of-00002.safetensors
+2, model.layers.33.mlp.down_proj.weight, torch.Size([2560, 8192]), layer 33, model-00002-of-00002.safetensors
+3, model.layers.33.mlp.gate_proj.weight, torch.Size([8192, 2560]), layer 33, model-00002-of-00002.safetensors
+3, model.layers.33.mlp.up_proj.weight, torch.Size([8192, 2560]), layer 33, model-00002-of-00002.safetensors
+3, model.layers.33.post_attention_layernorm.weight, torch.Size([2560]), layer 33, model-00002-of-00002.safetensors
+3, model.layers.33.self_attn.k_proj.weight, torch.Size([512, 2560]), layer 33, model-00002-of-00002.safetensors
+1, model.layers.33.self_attn.o_proj.weight, torch.Size([2560, 2560]), layer 33, model-00002-of-00002.safetensors
+3, model.layers.33.self_attn.q_proj.weight, torch.Size([2560, 2560]), layer 33, model-00002-of-00002.safetensors
+3, model.layers.33.self_attn.v_proj.weight, torch.Size([512, 2560]), layer 33, model-00002-of-00002.safetensors
+3, model.layers.34.input_layernorm.weight, torch.Size([2560]), layer 34, model-00002-of-00002.safetensors
+2, model.layers.34.mlp.down_proj.weight, torch.Size([2560, 8192]), layer 34, model-00002-of-00002.safetensors
+3, model.layers.34.mlp.gate_proj.weight, torch.Size([8192, 2560]), layer 34, model-00002-of-00002.safetensors
+3, model.layers.34.mlp.up_proj.weight, torch.Size([8192, 2560]), layer 34, model-00002-of-00002.safetensors
+3, model.layers.34.post_attention_layernorm.weight, torch.Size([2560]), layer 34, model-00002-of-00002.safetensors
+3, model.layers.34.self_attn.k_proj.weight, torch.Size([512, 2560]), layer 34, model-00002-of-00002.safetensors
+1, model.layers.34.self_attn.o_proj.weight, torch.Size([2560, 2560]), layer 34, model-00002-of-00002.safetensors
+3, model.layers.34.self_attn.q_proj.weight, torch.Size([2560, 2560]), layer 34, model-00002-of-00002.safetensors
+3, model.layers.34.self_attn.v_proj.weight, torch.Size([512, 2560]), layer 34, model-00002-of-00002.safetensors
+3, model.layers.35.input_layernorm.weight, torch.Size([2560]), layer 35, model-00002-of-00002.safetensors
+2, model.layers.35.mlp.down_proj.weight, torch.Size([2560, 8192]), layer 35, model-00002-of-00002.safetensors
+3, model.layers.35.mlp.gate_proj.weight, torch.Size([8192, 2560]), layer 35, model-00002-of-00002.safetensors
+3, model.layers.35.mlp.up_proj.weight, torch.Size([8192, 2560]), layer 35, model-00002-of-00002.safetensors
+3, model.layers.35.post_attention_layernorm.weight, torch.Size([2560]), layer 35, model-00002-of-00002.safetensors
+3, model.layers.35.self_attn.k_proj.weight, torch.Size([512, 2560]), layer 35, model-00002-of-00002.safetensors
+1, model.layers.35.self_attn.o_proj.weight, torch.Size([2560, 2560]), layer 35, model-00002-of-00002.safetensors
+3, model.layers.35.self_attn.q_proj.weight, torch.Size([2560, 2560]), layer 35, model-00002-of-00002.safetensors
+3, model.layers.35.self_attn.v_proj.weight, torch.Size([512, 2560]), layer 35, model-00002-of-00002.safetensors
+3, model.layers.36.input_layernorm.weight, torch.Size([2560]), layer 36, model-00002-of-00002.safetensors
+2, model.layers.36.mlp.down_proj.weight, torch.Size([2560, 8192]), layer 36, model-00002-of-00002.safetensors
+3, model.layers.36.mlp.gate_proj.weight, torch.Size([8192, 2560]), layer 36, model-00002-of-00002.safetensors
+3, model.layers.36.mlp.up_proj.weight, torch.Size([8192, 2560]), layer 36, model-00002-of-00002.safetensors
+3, model.layers.36.post_attention_layernorm.weight, torch.Size([2560]), layer 36, model-00002-of-00002.safetensors
+3, model.layers.36.self_attn.k_proj.weight, torch.Size([512, 2560]), layer 36, model-00002-of-00002.safetensors
+1, model.layers.36.self_attn.o_proj.weight, torch.Size([2560, 2560]), layer 36, model-00002-of-00002.safetensors
+3, model.layers.36.self_attn.q_proj.weight, torch.Size([2560, 2560]), layer 36, model-00002-of-00002.safetensors
+3, model.layers.36.self_attn.v_proj.weight, torch.Size([512, 2560]), layer 36, model-00002-of-00002.safetensors
+3, model.layers.37.input_layernorm.weight, torch.Size([2560]), layer 37, model-00002-of-00002.safetensors
+2, model.layers.37.mlp.down_proj.weight, torch.Size([2560, 8192]), layer 37, model-00002-of-00002.safetensors
+3, model.layers.37.mlp.gate_proj.weight, torch.Size([8192, 2560]), layer 37, model-00002-of-00002.safetensors
+3, model.layers.37.mlp.up_proj.weight, torch.Size([8192, 2560]), layer 37, model-00002-of-00002.safetensors
+3, model.layers.37.post_attention_layernorm.weight, torch.Size([2560]), layer 37, model-00002-of-00002.safetensors
+3, model.layers.37.self_attn.k_proj.weight, torch.Size([512, 2560]), layer 37, model-00002-of-00002.safetensors
+1, model.layers.37.self_attn.o_proj.weight, torch.Size([2560, 2560]), layer 37, model-00002-of-00002.safetensors
+3, model.layers.37.self_attn.q_proj.weight, torch.Size([2560, 2560]), layer 37, model-00002-of-00002.safetensors
+3, model.layers.37.self_attn.v_proj.weight, torch.Size([512, 2560]), layer 37, model-00002-of-00002.safetensors
+3, model.layers.38.input_layernorm.weight, torch.Size([2560]), layer 38, model-00002-of-00002.safetensors
+2, model.layers.38.mlp.down_proj.weight, torch.Size([2560, 8192]), layer 38, model-00002-of-00002.safetensors
+3, model.layers.38.mlp.gate_proj.weight, torch.Size([8192, 2560]), layer 38, model-00002-of-00002.safetensors
+3, model.layers.38.mlp.up_proj.weight, torch.Size([8192, 2560]), layer 38, model-00002-of-00002.safetensors
+3, model.layers.38.post_attention_layernorm.weight, torch.Size([2560]), layer 38, model-00002-of-00002.safetensors
+3, model.layers.38.self_attn.k_proj.weight, torch.Size([512, 2560]), layer 38, model-00002-of-00002.safetensors
+1, model.layers.38.self_attn.o_proj.weight, torch.Size([2560, 2560]), layer 38, model-00002-of-00002.safetensors
+3, model.layers.38.self_attn.q_proj.weight, torch.Size([2560, 2560]), layer 38, model-00002-of-00002.safetensors
+3, model.layers.38.self_attn.v_proj.weight, torch.Size([512, 2560]), layer 38, model-00002-of-00002.safetensors
+3, model.layers.39.input_layernorm.weight, torch.Size([2560]), layer 39, model-00002-of-00002.safetensors
+2, model.layers.39.mlp.down_proj.weight, torch.Size([2560, 8192]), layer 39, model-00002-of-00002.safetensors
+3, model.layers.39.mlp.gate_proj.weight, torch.Size([8192, 2560]), layer 39, model-00002-of-00002.safetensors
+3, model.layers.39.mlp.up_proj.weight, torch.Size([8192, 2560]), layer 39, model-00002-of-00002.safetensors
+3, model.layers.39.post_attention_layernorm.weight, torch.Size([2560]), layer 39, model-00002-of-00002.safetensors
+3, model.layers.39.self_attn.k_proj.weight, torch.Size([512, 2560]), layer 39, model-00002-of-00002.safetensors
+1, model.layers.39.self_attn.o_proj.weight, torch.Size([2560, 2560]), layer 39, model-00002-of-00002.safetensors
+3, model.layers.39.self_attn.q_proj.weight, torch.Size([2560, 2560]), layer 39, model-00002-of-00002.safetensors
+3, model.layers.39.self_attn.v_proj.weight, torch.Size([512, 2560]), layer 39, model-00002-of-00002.safetensors
+4, model.norm.weight, torch.Size([2560]), layer -1, model-00002-of-00002.safetensors

special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "<|end_of_text|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|end_of_text|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|pad|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<|unk|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,783 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "100256": {
+      "content": "<|pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100257": {
+      "content": "<|end_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100258": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100259": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100260": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100261": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100262": {
+      "content": "<|filename|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100263": {
+      "content": "<|reponame|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100264": {
+      "content": "<|start_of_role|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100265": {
+      "content": "<|end_of_role|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100266": {
+      "content": "<|unused_1|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100267": {
+      "content": "<|start_of_plugin|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100268": {
+      "content": "<|end_of_plugin|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100269": {
+      "content": "<|unk|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100270": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100271": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100272": {
+      "content": "<tool_response>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100273": {
+      "content": "</tool_response>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100274": {
+      "content": "<think>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100275": {
+      "content": "</think>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100276": {
+      "content": "<think_on>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100277": {
+      "content": "<think_off>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100278": {
+      "content": "<schema>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100279": {
+      "content": "</schema>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100280": {
+      "content": "<tools>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100281": {
+      "content": "</tools>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100282": {
+      "content": "<documents>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100283": {
+      "content": "</documents>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100284": {
+      "content": "<|unused_15|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100285": {
+      "content": "<|unused_16|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100286": {
+      "content": "<|unused_17|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100287": {
+      "content": "<|unused_18|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100288": {
+      "content": "<|unused_19|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100289": {
+      "content": "<|unused_20|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100290": {
+      "content": "<|unused_21|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100291": {
+      "content": "<|unused_22|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100292": {
+      "content": "<|unused_23|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100293": {
+      "content": "<|unused_24|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100294": {
+      "content": "<|unused_25|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100295": {
+      "content": "<|unused_26|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100296": {
+      "content": "<|unused_27|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100297": {
+      "content": "<|unused_28|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100298": {
+      "content": "<|unused_29|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100299": {
+      "content": "<|unused_30|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100300": {
+      "content": "<|unused_31|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100301": {
+      "content": "<|unused_32|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100302": {
+      "content": "<|unused_33|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100303": {
+      "content": "<|unused_34|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100304": {
+      "content": "<|unused_35|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100305": {
+      "content": "<|unused_36|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100306": {
+      "content": "<|unused_37|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100307": {
+      "content": "<|unused_38|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100308": {
+      "content": "<|unused_39|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100309": {
+      "content": "<|unused_40|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100310": {
+      "content": "<|unused_41|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100311": {
+      "content": "<|unused_42|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100312": {
+      "content": "<|unused_43|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100313": {
+      "content": "<|unused_44|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100314": {
+      "content": "<|unused_45|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100315": {
+      "content": "<|unused_46|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100316": {
+      "content": "<|unused_47|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100317": {
+      "content": "<|unused_48|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100318": {
+      "content": "<|unused_49|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100319": {
+      "content": "<|unused_50|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100320": {
+      "content": "<|unused_51|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100321": {
+      "content": "<|unused_52|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100322": {
+      "content": "<|unused_53|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100323": {
+      "content": "<|unused_54|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100324": {
+      "content": "<|unused_55|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100325": {
+      "content": "<|unused_56|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100326": {
+      "content": "<|unused_57|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100327": {
+      "content": "<|unused_58|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100328": {
+      "content": "<|unused_59|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100329": {
+      "content": "<|unused_60|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100330": {
+      "content": "<|unused_61|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100331": {
+      "content": "<|unused_62|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100332": {
+      "content": "<|unused_63|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100333": {
+      "content": "<|unused_64|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100334": {
+      "content": "<|unused_65|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100335": {
+      "content": "<|unused_66|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100336": {
+      "content": "<|unused_67|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100337": {
+      "content": "<|unused_68|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100338": {
+      "content": "<|unused_69|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100339": {
+      "content": "<|unused_70|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100340": {
+      "content": "<|unused_71|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100341": {
+      "content": "<|unused_72|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100342": {
+      "content": "<|unused_73|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100343": {
+      "content": "<|unused_74|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100344": {
+      "content": "<|unused_75|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100345": {
+      "content": "<|unused_76|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100346": {
+      "content": "<|unused_77|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100347": {
+      "content": "<|unused_78|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100348": {
+      "content": "<|unused_79|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100349": {
+      "content": "<|unused_80|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100350": {
+      "content": "<|unused_81|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100351": {
+      "content": "<|unused_82|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|end_of_text|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|end_of_text|>",
+  "extra_special_tokens": {},
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<|pad|>",
+  "padding_side": "left",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|unk|>"
+}