From be0b8de8875e96a57dd16be576ab8215b8a8dc25 Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Sun, 28 Jun 2026 22:21:21 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: chargoddard/llama2-22b Source: Original Platform --- .gitattributes | 35 +++ README.md | 27 ++ config.json | 26 ++ frankenllama_22b.py | 188 ++++++++++++++ generation_config.json | 7 + pytorch_model-00001-of-00005.bin | 3 + pytorch_model-00002-of-00005.bin | 3 + pytorch_model-00003-of-00005.bin | 3 + pytorch_model-00004-of-00005.bin | 3 + pytorch_model-00005-of-00005.bin | 3 + pytorch_model.bin.index.json | 410 +++++++++++++++++++++++++++++++ special_tokens_map.json | 24 ++ tokenizer.model | 3 + tokenizer_config.json | 34 +++ 14 files changed, 769 insertions(+) create mode 100644 .gitattributes create mode 100644 README.md create mode 100644 config.json create mode 100644 frankenllama_22b.py create mode 100644 generation_config.json create mode 100644 pytorch_model-00001-of-00005.bin create mode 100644 pytorch_model-00002-of-00005.bin create mode 100644 pytorch_model-00003-of-00005.bin create mode 100644 pytorch_model-00004-of-00005.bin create mode 100644 pytorch_model-00005-of-00005.bin create mode 100644 pytorch_model.bin.index.json create mode 100644 special_tokens_map.json create mode 100644 tokenizer.model create mode 100644 tokenizer_config.json diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..a6344aa --- /dev/null +++ b/.gitattributes @@ -0,0 +1,35 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..5490ccb --- /dev/null +++ b/README.md @@ -0,0 +1,27 @@ +--- +model_type: llama +pipeline_tag: text-generation +datasets: +- togethercomputer/RedPajama-Data-1T-Sample +tags: +- llama +--- + +This is [Llama 2 13b](https://huggingface.co/meta-llama/Llama-2-13b-hf) with some additional attention heads from original-flavor Llama 33b frankensteined on. + +Fine-tuned on ~10M tokens from RedPajama to settle in the transplants a little. + +Not intended for use as-is - this model is meant to serve as a base for further tuning, hopefully with a greater capacity for learning than 13b. +# [Open LLM Leaderboard Evaluation Results](https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard) +Detailed results can be found [here](https://huggingface.co/datasets/open-llm-leaderboard/details_chargoddard__llama2-22b) + +| Metric | Value | +|-----------------------|---------------------------| +| Avg. | 46.85 | +| ARC (25-shot) | 58.53 | +| HellaSwag (10-shot) | 82.55 | +| MMLU (5-shot) | 54.68 | +| TruthfulQA (0-shot) | 39.84 | +| Winogrande (5-shot) | 76.32 | +| GSM8K (5-shot) | 9.93 | +| DROP (3-shot) | 6.08 | diff --git a/config.json b/config.json new file mode 100644 index 0000000..6e481a1 --- /dev/null +++ b/config.json @@ -0,0 +1,26 @@ +{ + "_name_or_path": "/root/llama2-22b/", + "architectures": [ + "LlamaForCausalLM" + ], + "bos_token_id": 1, + "eos_token_id": 2, + "hidden_act": "silu", + "hidden_size": 6656, + "initializer_range": 0.02, + "intermediate_size": 17920, + "max_position_embeddings": 4098, + "model_type": "llama", + "num_attention_heads": 52, + "num_hidden_layers": 40, + "num_key_value_heads": 52, + "pad_token_id": 0, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_scaling": null, + "tie_word_embeddings": false, + "torch_dtype": "float16", + "transformers_version": "4.32.0.dev0", + "use_cache": false, + "vocab_size": 32000 +} diff --git a/frankenllama_22b.py b/frankenllama_22b.py new file mode 100644 index 0000000..96b85eb --- /dev/null +++ b/frankenllama_22b.py @@ -0,0 +1,188 @@ +#!/usr/bin/env python3 +# Charles O. Goddard +# 7/20/2023 +"""Script used to generate the base frankenmerge. Output will need fine-tuning to be useful.""" + +import copy +import torch +from torch import Tensor, nn +import transformers + +from transformers.models.llama.modeling_llama import ( + LlamaForCausalLM, + LlamaDecoderLayer, +) +from transformers import LlamaForCausalLM, LlamaConfig + +import torch +import transformers +import numpy as np + + +MODEL_NAME_13B = "meta-llama/Llama-2-13b-hf" # primary model +MODEL_NAME_33B = "huggyllama/llama-30b" # donor +BLOCK_DIAGONAL = True +# If BLOCK_DIAGONAL is set to True, each tensor in the resultant model will form a +# block diagonal matrix, as illustrated below: + +# a a a 0 0 +# a a a 0 0 +# a a a 0 0 +# 0 0 0 b b +# 0 0 0 b b + +# In this configuration, the states (hidden and intermediate) from the original +# and donor models are completely decoupled. That is, the hidden states +# corresponding to the original model remain unchanged, and the new dimensions +# added from the donor model do not depend on the hidden states of the original model. + +# If BLOCK_DIAGONAL is set to False, the tensors will instead have the following form: + +# a a a 0 0 +# a a a 0 0 +# a a a 0 0 +# b b b b b +# b b b b b + +# In this case, the output of the newly added attention heads depends on the hidden +# state values as if they were part of the donor model. Although the original model's +# hidden states remain unchanged in either case, interaction between the new and old +# features will result in features of varying usefulness. + + +class NoInit: + def __enter__(self): + def noop(*args, **kwargs): + pass + + (k, u, n) = ( + torch.nn.init.kaiming_uniform_, + torch.nn.init.uniform_, + torch.nn.init.normal_, + ) + torch.nn.init.kaiming_uniform_ = noop + torch.nn.init.uniform_ = noop + torch.nn.init.normal_ = noop + + transformers.modeling_utils._init_weights = False + self.funcs = (k, u, n) + + def __exit__(self, *args): + (k, u, n) = self.funcs + ( + torch.nn.init.kaiming_uniform_, + torch.nn.init.uniform_, + torch.nn.init.normal_, + ) = ( + k, + u, + n, + ) + transformers.modeling_utils._init_weights = True + + +def format_kmb(n, digits=None): + n = int(n) + if n < 1000: + return str(n) + elif n < 1000_000: + return f"{round(n/1000, digits)}k" + elif n < 1000 * 1000 * 1000: + return f"{round(n/(1000*1000), digits)}m" + else: + return f"{round(n/(1000*1000*1000), digits)}b" + + +def count_params(model): + model_parameters = filter(lambda p: p.requires_grad, model.parameters()) + params = sum([np.prod(p.size()) for p in model_parameters]) + return int(params) + + +torch.set_default_dtype(torch.float16) + +config_13b: LlamaConfig = LlamaConfig.from_pretrained(MODEL_NAME_13B) +config_33b: LlamaConfig = LlamaConfig.from_pretrained(MODEL_NAME_33B) +config_more = copy.deepcopy(config_13b) +config_more.intermediate_size = config_33b.intermediate_size +config_more.hidden_size = config_33b.hidden_size +config_more.num_key_value_heads = config_33b.num_key_value_heads +config_more.num_attention_heads = config_33b.num_key_value_heads + +print(config_more) + +with NoInit(): + model = LlamaForCausalLM(config_more) + +print(f"{format_kmb(count_params(model), 3)} parameters") + + +def merge_tensors_inplace(dest: Tensor, s0: Tensor, s1: Tensor, block_diagonal: bool): + dest.zero_() + if block_diagonal: + dest[s0.shape[0] :, s0.shape[1] :] = s1[ + s0.shape[0] : dest.shape[0], + s0.shape[1] : dest.shape[1], + ] + else: + dest[s0.shape[0] :, :] = s1[ + s0.shape[0] : dest.shape[0], + : dest.shape[1], + ] + dest[: s0.shape[0], : s0.shape[1]] = s0 + + +with NoInit(): + donor_13b = ( + LlamaForCausalLM.from_pretrained(MODEL_NAME_13B).to(torch.float16).eval() + ) + donor_33b = ( + LlamaForCausalLM.from_pretrained(MODEL_NAME_33B).to(torch.float16).eval() + ) + +with torch.no_grad(): + for layer_idx in range(len(model.model.layers)): + layer: LlamaDecoderLayer = model.model.layers[layer_idx] + l13: LlamaDecoderLayer = donor_13b.model.layers[layer_idx] + l33: LlamaDecoderLayer = donor_33b.model.layers[layer_idx] + + for name in ("q_proj", "k_proj", "v_proj", "o_proj"): + dest: nn.Linear = getattr(layer.self_attn, name) + s13: nn.Linear = getattr(l13.self_attn, name) + s33: nn.Linear = getattr(l33.self_attn, name) + merge_tensors_inplace(dest.weight, s13.weight, s33.weight, BLOCK_DIAGONAL) + + for name in ("up_proj", "gate_proj", "down_proj"): + dest: nn.Linear = getattr(layer.mlp, name) + s13: nn.Linear = getattr(l13.mlp, name) + s33: nn.Linear = getattr(l33.mlp, name) + merge_tensors_inplace(dest.weight, s13.weight, s33.weight, BLOCK_DIAGONAL) + + layer.input_layernorm.weight[:] = l33.input_layernorm.weight[ + : layer.input_layernorm.weight.shape[0] + ] + layer.input_layernorm.weight[ + : l13.input_layernorm.weight.shape[0] + ] = l13.input_layernorm.weight + layer.post_attention_layernorm.weight[:] = l33.post_attention_layernorm.weight[ + : layer.post_attention_layernorm.weight.shape[0] + ] + layer.post_attention_layernorm.weight[ + : l13.post_attention_layernorm.weight.shape[0] + ] = l13.post_attention_layernorm.weight + + # have initial output depend on only original llama2-13b features, so model + # starts unimpaired and can learn to incorporate the new features as well + model.lm_head.weight.zero_() + model.lm_head.weight[ + : donor_13b.lm_head.weight.shape[0], : donor_13b.lm_head.weight.shape[1] + ] = donor_13b.lm_head.weight + + merge_tensors_inplace( + model.model.embed_tokens.weight, + donor_13b.model.embed_tokens.weight, + donor_33b.model.embed_tokens.weight, + BLOCK_DIAGONAL, + ) + +model.save_pretrained("./llama2-22b/", safe_serialization=True) diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..8c88a8f --- /dev/null +++ b/generation_config.json @@ -0,0 +1,7 @@ +{ + "_from_model_config": true, + "bos_token_id": 1, + "eos_token_id": 2, + "pad_token_id": 0, + "transformers_version": "4.32.0.dev0" +} diff --git a/pytorch_model-00001-of-00005.bin b/pytorch_model-00001-of-00005.bin new file mode 100644 index 0000000..d922195 --- /dev/null +++ b/pytorch_model-00001-of-00005.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:70019781140edfa10f01730a1e20b649a82cd104981bbd42eb175c985c581741 +size 9818324691 diff --git a/pytorch_model-00002-of-00005.bin b/pytorch_model-00002-of-00005.bin new file mode 100644 index 0000000..6609546 --- /dev/null +++ b/pytorch_model-00002-of-00005.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8d96b90f05e26d973b86c512c33b121683c30efd88eb6ee2bca08fae7a128366 +size 9958102871 diff --git a/pytorch_model-00003-of-00005.bin b/pytorch_model-00003-of-00005.bin new file mode 100644 index 0000000..8d49d0a --- /dev/null +++ b/pytorch_model-00003-of-00005.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dd2b8673c387c5497edbeee036c9fb6072f0101deb7fb97158fc53cb65595ed0 +size 9896734715 diff --git a/pytorch_model-00004-of-00005.bin b/pytorch_model-00004-of-00005.bin new file mode 100644 index 0000000..7531c28 --- /dev/null +++ b/pytorch_model-00004-of-00005.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b60d983a87dd4f8450583eedda634dc452152a719ea6ce86bbd77548dd82240d +size 9869470609 diff --git a/pytorch_model-00005-of-00005.bin b/pytorch_model-00005-of-00005.bin new file mode 100644 index 0000000..59388b0 --- /dev/null +++ b/pytorch_model-00005-of-00005.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d950dcd51ecfdc5cc77c357acb2dd2766cbda8c538ab6e333f62abef67feac0c +size 4113434135 diff --git a/pytorch_model.bin.index.json b/pytorch_model.bin.index.json new file mode 100644 index 0000000..8c93daa --- /dev/null +++ b/pytorch_model.bin.index.json @@ -0,0 +1,410 @@ +{ + "metadata": { + "total_size": 43655923712 + }, + "weight_map": { + "lm_head.weight": "pytorch_model-00005-of-00005.bin", + "model.embed_tokens.weight": "pytorch_model-00001-of-00005.bin", + "model.layers.0.input_layernorm.weight": "pytorch_model-00001-of-00005.bin", + "model.layers.0.mlp.down_proj.weight": "pytorch_model-00001-of-00005.bin", + "model.layers.0.mlp.gate_proj.weight": "pytorch_model-00001-of-00005.bin", + "model.layers.0.mlp.up_proj.weight": "pytorch_model-00001-of-00005.bin", + "model.layers.0.post_attention_layernorm.weight": "pytorch_model-00001-of-00005.bin", + "model.layers.0.self_attn.k_proj.weight": "pytorch_model-00001-of-00005.bin", + "model.layers.0.self_attn.o_proj.weight": "pytorch_model-00001-of-00005.bin", + "model.layers.0.self_attn.q_proj.weight": "pytorch_model-00001-of-00005.bin", + "model.layers.0.self_attn.rotary_emb.inv_freq": "pytorch_model-00001-of-00005.bin", + "model.layers.0.self_attn.v_proj.weight": "pytorch_model-00001-of-00005.bin", + "model.layers.1.input_layernorm.weight": "pytorch_model-00001-of-00005.bin", + "model.layers.1.mlp.down_proj.weight": "pytorch_model-00001-of-00005.bin", + "model.layers.1.mlp.gate_proj.weight": "pytorch_model-00001-of-00005.bin", + "model.layers.1.mlp.up_proj.weight": "pytorch_model-00001-of-00005.bin", + "model.layers.1.post_attention_layernorm.weight": "pytorch_model-00001-of-00005.bin", + "model.layers.1.self_attn.k_proj.weight": "pytorch_model-00001-of-00005.bin", + "model.layers.1.self_attn.o_proj.weight": "pytorch_model-00001-of-00005.bin", + "model.layers.1.self_attn.q_proj.weight": "pytorch_model-00001-of-00005.bin", + "model.layers.1.self_attn.rotary_emb.inv_freq": "pytorch_model-00001-of-00005.bin", + "model.layers.1.self_attn.v_proj.weight": "pytorch_model-00001-of-00005.bin", + "model.layers.10.input_layernorm.weight": "pytorch_model-00002-of-00005.bin", + "model.layers.10.mlp.down_proj.weight": "pytorch_model-00002-of-00005.bin", + "model.layers.10.mlp.gate_proj.weight": "pytorch_model-00002-of-00005.bin", + "model.layers.10.mlp.up_proj.weight": "pytorch_model-00002-of-00005.bin", + "model.layers.10.post_attention_layernorm.weight": "pytorch_model-00002-of-00005.bin", + "model.layers.10.self_attn.k_proj.weight": "pytorch_model-00002-of-00005.bin", + "model.layers.10.self_attn.o_proj.weight": "pytorch_model-00002-of-00005.bin", + "model.layers.10.self_attn.q_proj.weight": "pytorch_model-00002-of-00005.bin", + "model.layers.10.self_attn.rotary_emb.inv_freq": "pytorch_model-00002-of-00005.bin", + "model.layers.10.self_attn.v_proj.weight": "pytorch_model-00002-of-00005.bin", + "model.layers.11.input_layernorm.weight": "pytorch_model-00002-of-00005.bin", + "model.layers.11.mlp.down_proj.weight": "pytorch_model-00002-of-00005.bin", + "model.layers.11.mlp.gate_proj.weight": "pytorch_model-00002-of-00005.bin", + "model.layers.11.mlp.up_proj.weight": "pytorch_model-00002-of-00005.bin", + "model.layers.11.post_attention_layernorm.weight": "pytorch_model-00002-of-00005.bin", + "model.layers.11.self_attn.k_proj.weight": "pytorch_model-00002-of-00005.bin", + "model.layers.11.self_attn.o_proj.weight": "pytorch_model-00002-of-00005.bin", + "model.layers.11.self_attn.q_proj.weight": "pytorch_model-00002-of-00005.bin", + "model.layers.11.self_attn.rotary_emb.inv_freq": "pytorch_model-00002-of-00005.bin", + "model.layers.11.self_attn.v_proj.weight": "pytorch_model-00002-of-00005.bin", + "model.layers.12.input_layernorm.weight": "pytorch_model-00002-of-00005.bin", + "model.layers.12.mlp.down_proj.weight": "pytorch_model-00002-of-00005.bin", + "model.layers.12.mlp.gate_proj.weight": "pytorch_model-00002-of-00005.bin", + "model.layers.12.mlp.up_proj.weight": "pytorch_model-00002-of-00005.bin", + "model.layers.12.post_attention_layernorm.weight": "pytorch_model-00002-of-00005.bin", + "model.layers.12.self_attn.k_proj.weight": "pytorch_model-00002-of-00005.bin", + "model.layers.12.self_attn.o_proj.weight": "pytorch_model-00002-of-00005.bin", + "model.layers.12.self_attn.q_proj.weight": "pytorch_model-00002-of-00005.bin", + "model.layers.12.self_attn.rotary_emb.inv_freq": "pytorch_model-00002-of-00005.bin", + "model.layers.12.self_attn.v_proj.weight": "pytorch_model-00002-of-00005.bin", + "model.layers.13.input_layernorm.weight": "pytorch_model-00002-of-00005.bin", + "model.layers.13.mlp.down_proj.weight": "pytorch_model-00002-of-00005.bin", + "model.layers.13.mlp.gate_proj.weight": "pytorch_model-00002-of-00005.bin", + "model.layers.13.mlp.up_proj.weight": "pytorch_model-00002-of-00005.bin", + "model.layers.13.post_attention_layernorm.weight": "pytorch_model-00002-of-00005.bin", + "model.layers.13.self_attn.k_proj.weight": "pytorch_model-00002-of-00005.bin", + "model.layers.13.self_attn.o_proj.weight": "pytorch_model-00002-of-00005.bin", + "model.layers.13.self_attn.q_proj.weight": "pytorch_model-00002-of-00005.bin", + "model.layers.13.self_attn.rotary_emb.inv_freq": "pytorch_model-00002-of-00005.bin", + "model.layers.13.self_attn.v_proj.weight": "pytorch_model-00002-of-00005.bin", + "model.layers.14.input_layernorm.weight": "pytorch_model-00002-of-00005.bin", + "model.layers.14.mlp.down_proj.weight": "pytorch_model-00002-of-00005.bin", + "model.layers.14.mlp.gate_proj.weight": "pytorch_model-00002-of-00005.bin", + "model.layers.14.mlp.up_proj.weight": "pytorch_model-00002-of-00005.bin", + "model.layers.14.post_attention_layernorm.weight": "pytorch_model-00002-of-00005.bin", + "model.layers.14.self_attn.k_proj.weight": "pytorch_model-00002-of-00005.bin", + "model.layers.14.self_attn.o_proj.weight": "pytorch_model-00002-of-00005.bin", + "model.layers.14.self_attn.q_proj.weight": "pytorch_model-00002-of-00005.bin", + "model.layers.14.self_attn.rotary_emb.inv_freq": "pytorch_model-00002-of-00005.bin", + "model.layers.14.self_attn.v_proj.weight": "pytorch_model-00002-of-00005.bin", + "model.layers.15.input_layernorm.weight": "pytorch_model-00002-of-00005.bin", + "model.layers.15.mlp.down_proj.weight": "pytorch_model-00002-of-00005.bin", + "model.layers.15.mlp.gate_proj.weight": "pytorch_model-00002-of-00005.bin", + "model.layers.15.mlp.up_proj.weight": "pytorch_model-00002-of-00005.bin", + "model.layers.15.post_attention_layernorm.weight": "pytorch_model-00002-of-00005.bin", + "model.layers.15.self_attn.k_proj.weight": "pytorch_model-00002-of-00005.bin", + "model.layers.15.self_attn.o_proj.weight": "pytorch_model-00002-of-00005.bin", + "model.layers.15.self_attn.q_proj.weight": "pytorch_model-00002-of-00005.bin", + "model.layers.15.self_attn.rotary_emb.inv_freq": "pytorch_model-00002-of-00005.bin", + "model.layers.15.self_attn.v_proj.weight": "pytorch_model-00002-of-00005.bin", + "model.layers.16.input_layernorm.weight": "pytorch_model-00002-of-00005.bin", + "model.layers.16.mlp.down_proj.weight": "pytorch_model-00002-of-00005.bin", + "model.layers.16.mlp.gate_proj.weight": "pytorch_model-00002-of-00005.bin", + "model.layers.16.mlp.up_proj.weight": "pytorch_model-00002-of-00005.bin", + "model.layers.16.post_attention_layernorm.weight": "pytorch_model-00002-of-00005.bin", + "model.layers.16.self_attn.k_proj.weight": "pytorch_model-00002-of-00005.bin", + "model.layers.16.self_attn.o_proj.weight": "pytorch_model-00002-of-00005.bin", + "model.layers.16.self_attn.q_proj.weight": "pytorch_model-00002-of-00005.bin", + "model.layers.16.self_attn.rotary_emb.inv_freq": "pytorch_model-00002-of-00005.bin", + "model.layers.16.self_attn.v_proj.weight": "pytorch_model-00002-of-00005.bin", + "model.layers.17.input_layernorm.weight": "pytorch_model-00002-of-00005.bin", + "model.layers.17.mlp.down_proj.weight": "pytorch_model-00002-of-00005.bin", + "model.layers.17.mlp.gate_proj.weight": "pytorch_model-00002-of-00005.bin", + "model.layers.17.mlp.up_proj.weight": "pytorch_model-00002-of-00005.bin", + "model.layers.17.post_attention_layernorm.weight": "pytorch_model-00002-of-00005.bin", + "model.layers.17.self_attn.k_proj.weight": "pytorch_model-00002-of-00005.bin", + "model.layers.17.self_attn.o_proj.weight": "pytorch_model-00002-of-00005.bin", + "model.layers.17.self_attn.q_proj.weight": "pytorch_model-00002-of-00005.bin", + "model.layers.17.self_attn.rotary_emb.inv_freq": "pytorch_model-00002-of-00005.bin", + "model.layers.17.self_attn.v_proj.weight": "pytorch_model-00002-of-00005.bin", + "model.layers.18.input_layernorm.weight": "pytorch_model-00003-of-00005.bin", + "model.layers.18.mlp.down_proj.weight": "pytorch_model-00003-of-00005.bin", + "model.layers.18.mlp.gate_proj.weight": "pytorch_model-00003-of-00005.bin", + "model.layers.18.mlp.up_proj.weight": "pytorch_model-00003-of-00005.bin", + "model.layers.18.post_attention_layernorm.weight": "pytorch_model-00003-of-00005.bin", + "model.layers.18.self_attn.k_proj.weight": "pytorch_model-00003-of-00005.bin", + "model.layers.18.self_attn.o_proj.weight": "pytorch_model-00003-of-00005.bin", + "model.layers.18.self_attn.q_proj.weight": "pytorch_model-00002-of-00005.bin", + "model.layers.18.self_attn.rotary_emb.inv_freq": "pytorch_model-00003-of-00005.bin", + "model.layers.18.self_attn.v_proj.weight": "pytorch_model-00003-of-00005.bin", + "model.layers.19.input_layernorm.weight": "pytorch_model-00003-of-00005.bin", + "model.layers.19.mlp.down_proj.weight": "pytorch_model-00003-of-00005.bin", + "model.layers.19.mlp.gate_proj.weight": "pytorch_model-00003-of-00005.bin", + "model.layers.19.mlp.up_proj.weight": "pytorch_model-00003-of-00005.bin", + "model.layers.19.post_attention_layernorm.weight": "pytorch_model-00003-of-00005.bin", + "model.layers.19.self_attn.k_proj.weight": "pytorch_model-00003-of-00005.bin", + "model.layers.19.self_attn.o_proj.weight": "pytorch_model-00003-of-00005.bin", + "model.layers.19.self_attn.q_proj.weight": "pytorch_model-00003-of-00005.bin", + "model.layers.19.self_attn.rotary_emb.inv_freq": "pytorch_model-00003-of-00005.bin", + "model.layers.19.self_attn.v_proj.weight": "pytorch_model-00003-of-00005.bin", + "model.layers.2.input_layernorm.weight": "pytorch_model-00001-of-00005.bin", + "model.layers.2.mlp.down_proj.weight": "pytorch_model-00001-of-00005.bin", + "model.layers.2.mlp.gate_proj.weight": "pytorch_model-00001-of-00005.bin", + "model.layers.2.mlp.up_proj.weight": "pytorch_model-00001-of-00005.bin", + "model.layers.2.post_attention_layernorm.weight": "pytorch_model-00001-of-00005.bin", + "model.layers.2.self_attn.k_proj.weight": "pytorch_model-00001-of-00005.bin", + "model.layers.2.self_attn.o_proj.weight": "pytorch_model-00001-of-00005.bin", + "model.layers.2.self_attn.q_proj.weight": "pytorch_model-00001-of-00005.bin", + "model.layers.2.self_attn.rotary_emb.inv_freq": "pytorch_model-00001-of-00005.bin", + "model.layers.2.self_attn.v_proj.weight": "pytorch_model-00001-of-00005.bin", + "model.layers.20.input_layernorm.weight": "pytorch_model-00003-of-00005.bin", + "model.layers.20.mlp.down_proj.weight": "pytorch_model-00003-of-00005.bin", + "model.layers.20.mlp.gate_proj.weight": "pytorch_model-00003-of-00005.bin", + "model.layers.20.mlp.up_proj.weight": "pytorch_model-00003-of-00005.bin", + "model.layers.20.post_attention_layernorm.weight": "pytorch_model-00003-of-00005.bin", + "model.layers.20.self_attn.k_proj.weight": "pytorch_model-00003-of-00005.bin", + "model.layers.20.self_attn.o_proj.weight": "pytorch_model-00003-of-00005.bin", + "model.layers.20.self_attn.q_proj.weight": "pytorch_model-00003-of-00005.bin", + "model.layers.20.self_attn.rotary_emb.inv_freq": "pytorch_model-00003-of-00005.bin", + "model.layers.20.self_attn.v_proj.weight": "pytorch_model-00003-of-00005.bin", + "model.layers.21.input_layernorm.weight": "pytorch_model-00003-of-00005.bin", + "model.layers.21.mlp.down_proj.weight": "pytorch_model-00003-of-00005.bin", + "model.layers.21.mlp.gate_proj.weight": "pytorch_model-00003-of-00005.bin", + "model.layers.21.mlp.up_proj.weight": "pytorch_model-00003-of-00005.bin", + "model.layers.21.post_attention_layernorm.weight": "pytorch_model-00003-of-00005.bin", + "model.layers.21.self_attn.k_proj.weight": "pytorch_model-00003-of-00005.bin", + "model.layers.21.self_attn.o_proj.weight": "pytorch_model-00003-of-00005.bin", + "model.layers.21.self_attn.q_proj.weight": "pytorch_model-00003-of-00005.bin", + "model.layers.21.self_attn.rotary_emb.inv_freq": "pytorch_model-00003-of-00005.bin", + "model.layers.21.self_attn.v_proj.weight": "pytorch_model-00003-of-00005.bin", + "model.layers.22.input_layernorm.weight": "pytorch_model-00003-of-00005.bin", + "model.layers.22.mlp.down_proj.weight": "pytorch_model-00003-of-00005.bin", + "model.layers.22.mlp.gate_proj.weight": "pytorch_model-00003-of-00005.bin", + "model.layers.22.mlp.up_proj.weight": "pytorch_model-00003-of-00005.bin", + "model.layers.22.post_attention_layernorm.weight": "pytorch_model-00003-of-00005.bin", + "model.layers.22.self_attn.k_proj.weight": "pytorch_model-00003-of-00005.bin", + "model.layers.22.self_attn.o_proj.weight": "pytorch_model-00003-of-00005.bin", + "model.layers.22.self_attn.q_proj.weight": "pytorch_model-00003-of-00005.bin", + "model.layers.22.self_attn.rotary_emb.inv_freq": "pytorch_model-00003-of-00005.bin", + "model.layers.22.self_attn.v_proj.weight": "pytorch_model-00003-of-00005.bin", + "model.layers.23.input_layernorm.weight": "pytorch_model-00003-of-00005.bin", + "model.layers.23.mlp.down_proj.weight": "pytorch_model-00003-of-00005.bin", + "model.layers.23.mlp.gate_proj.weight": "pytorch_model-00003-of-00005.bin", + "model.layers.23.mlp.up_proj.weight": "pytorch_model-00003-of-00005.bin", + "model.layers.23.post_attention_layernorm.weight": "pytorch_model-00003-of-00005.bin", + "model.layers.23.self_attn.k_proj.weight": "pytorch_model-00003-of-00005.bin", + "model.layers.23.self_attn.o_proj.weight": "pytorch_model-00003-of-00005.bin", + "model.layers.23.self_attn.q_proj.weight": "pytorch_model-00003-of-00005.bin", + "model.layers.23.self_attn.rotary_emb.inv_freq": "pytorch_model-00003-of-00005.bin", + "model.layers.23.self_attn.v_proj.weight": "pytorch_model-00003-of-00005.bin", + "model.layers.24.input_layernorm.weight": "pytorch_model-00003-of-00005.bin", + "model.layers.24.mlp.down_proj.weight": "pytorch_model-00003-of-00005.bin", + "model.layers.24.mlp.gate_proj.weight": "pytorch_model-00003-of-00005.bin", + "model.layers.24.mlp.up_proj.weight": "pytorch_model-00003-of-00005.bin", + "model.layers.24.post_attention_layernorm.weight": "pytorch_model-00003-of-00005.bin", + "model.layers.24.self_attn.k_proj.weight": "pytorch_model-00003-of-00005.bin", + "model.layers.24.self_attn.o_proj.weight": "pytorch_model-00003-of-00005.bin", + "model.layers.24.self_attn.q_proj.weight": "pytorch_model-00003-of-00005.bin", + "model.layers.24.self_attn.rotary_emb.inv_freq": "pytorch_model-00003-of-00005.bin", + "model.layers.24.self_attn.v_proj.weight": "pytorch_model-00003-of-00005.bin", + "model.layers.25.input_layernorm.weight": "pytorch_model-00003-of-00005.bin", + "model.layers.25.mlp.down_proj.weight": "pytorch_model-00003-of-00005.bin", + "model.layers.25.mlp.gate_proj.weight": "pytorch_model-00003-of-00005.bin", + "model.layers.25.mlp.up_proj.weight": "pytorch_model-00003-of-00005.bin", + "model.layers.25.post_attention_layernorm.weight": "pytorch_model-00003-of-00005.bin", + "model.layers.25.self_attn.k_proj.weight": "pytorch_model-00003-of-00005.bin", + "model.layers.25.self_attn.o_proj.weight": "pytorch_model-00003-of-00005.bin", + "model.layers.25.self_attn.q_proj.weight": "pytorch_model-00003-of-00005.bin", + "model.layers.25.self_attn.rotary_emb.inv_freq": "pytorch_model-00003-of-00005.bin", + "model.layers.25.self_attn.v_proj.weight": "pytorch_model-00003-of-00005.bin", + "model.layers.26.input_layernorm.weight": "pytorch_model-00003-of-00005.bin", + "model.layers.26.mlp.down_proj.weight": "pytorch_model-00003-of-00005.bin", + "model.layers.26.mlp.gate_proj.weight": "pytorch_model-00003-of-00005.bin", + "model.layers.26.mlp.up_proj.weight": "pytorch_model-00003-of-00005.bin", + "model.layers.26.post_attention_layernorm.weight": "pytorch_model-00003-of-00005.bin", + "model.layers.26.self_attn.k_proj.weight": "pytorch_model-00003-of-00005.bin", + "model.layers.26.self_attn.o_proj.weight": "pytorch_model-00003-of-00005.bin", + "model.layers.26.self_attn.q_proj.weight": "pytorch_model-00003-of-00005.bin", + "model.layers.26.self_attn.rotary_emb.inv_freq": "pytorch_model-00003-of-00005.bin", + "model.layers.26.self_attn.v_proj.weight": "pytorch_model-00003-of-00005.bin", + "model.layers.27.input_layernorm.weight": "pytorch_model-00004-of-00005.bin", + "model.layers.27.mlp.down_proj.weight": "pytorch_model-00004-of-00005.bin", + "model.layers.27.mlp.gate_proj.weight": "pytorch_model-00004-of-00005.bin", + "model.layers.27.mlp.up_proj.weight": "pytorch_model-00004-of-00005.bin", + "model.layers.27.post_attention_layernorm.weight": "pytorch_model-00004-of-00005.bin", + "model.layers.27.self_attn.k_proj.weight": "pytorch_model-00003-of-00005.bin", + "model.layers.27.self_attn.o_proj.weight": "pytorch_model-00003-of-00005.bin", + "model.layers.27.self_attn.q_proj.weight": "pytorch_model-00003-of-00005.bin", + "model.layers.27.self_attn.rotary_emb.inv_freq": "pytorch_model-00003-of-00005.bin", + "model.layers.27.self_attn.v_proj.weight": "pytorch_model-00003-of-00005.bin", + "model.layers.28.input_layernorm.weight": "pytorch_model-00004-of-00005.bin", + "model.layers.28.mlp.down_proj.weight": "pytorch_model-00004-of-00005.bin", + "model.layers.28.mlp.gate_proj.weight": "pytorch_model-00004-of-00005.bin", + "model.layers.28.mlp.up_proj.weight": "pytorch_model-00004-of-00005.bin", + "model.layers.28.post_attention_layernorm.weight": "pytorch_model-00004-of-00005.bin", + "model.layers.28.self_attn.k_proj.weight": "pytorch_model-00004-of-00005.bin", + "model.layers.28.self_attn.o_proj.weight": "pytorch_model-00004-of-00005.bin", + "model.layers.28.self_attn.q_proj.weight": "pytorch_model-00004-of-00005.bin", + "model.layers.28.self_attn.rotary_emb.inv_freq": "pytorch_model-00004-of-00005.bin", + "model.layers.28.self_attn.v_proj.weight": "pytorch_model-00004-of-00005.bin", + "model.layers.29.input_layernorm.weight": "pytorch_model-00004-of-00005.bin", + "model.layers.29.mlp.down_proj.weight": "pytorch_model-00004-of-00005.bin", + "model.layers.29.mlp.gate_proj.weight": "pytorch_model-00004-of-00005.bin", + "model.layers.29.mlp.up_proj.weight": "pytorch_model-00004-of-00005.bin", + "model.layers.29.post_attention_layernorm.weight": "pytorch_model-00004-of-00005.bin", + "model.layers.29.self_attn.k_proj.weight": "pytorch_model-00004-of-00005.bin", + "model.layers.29.self_attn.o_proj.weight": "pytorch_model-00004-of-00005.bin", + "model.layers.29.self_attn.q_proj.weight": "pytorch_model-00004-of-00005.bin", + "model.layers.29.self_attn.rotary_emb.inv_freq": "pytorch_model-00004-of-00005.bin", + "model.layers.29.self_attn.v_proj.weight": "pytorch_model-00004-of-00005.bin", + "model.layers.3.input_layernorm.weight": "pytorch_model-00001-of-00005.bin", + "model.layers.3.mlp.down_proj.weight": "pytorch_model-00001-of-00005.bin", + "model.layers.3.mlp.gate_proj.weight": "pytorch_model-00001-of-00005.bin", + "model.layers.3.mlp.up_proj.weight": "pytorch_model-00001-of-00005.bin", + "model.layers.3.post_attention_layernorm.weight": "pytorch_model-00001-of-00005.bin", + "model.layers.3.self_attn.k_proj.weight": "pytorch_model-00001-of-00005.bin", + "model.layers.3.self_attn.o_proj.weight": "pytorch_model-00001-of-00005.bin", + "model.layers.3.self_attn.q_proj.weight": "pytorch_model-00001-of-00005.bin", + "model.layers.3.self_attn.rotary_emb.inv_freq": "pytorch_model-00001-of-00005.bin", + "model.layers.3.self_attn.v_proj.weight": "pytorch_model-00001-of-00005.bin", + "model.layers.30.input_layernorm.weight": "pytorch_model-00004-of-00005.bin", + "model.layers.30.mlp.down_proj.weight": "pytorch_model-00004-of-00005.bin", + "model.layers.30.mlp.gate_proj.weight": "pytorch_model-00004-of-00005.bin", + "model.layers.30.mlp.up_proj.weight": "pytorch_model-00004-of-00005.bin", + "model.layers.30.post_attention_layernorm.weight": "pytorch_model-00004-of-00005.bin", + "model.layers.30.self_attn.k_proj.weight": "pytorch_model-00004-of-00005.bin", + "model.layers.30.self_attn.o_proj.weight": "pytorch_model-00004-of-00005.bin", + "model.layers.30.self_attn.q_proj.weight": "pytorch_model-00004-of-00005.bin", + "model.layers.30.self_attn.rotary_emb.inv_freq": "pytorch_model-00004-of-00005.bin", + "model.layers.30.self_attn.v_proj.weight": "pytorch_model-00004-of-00005.bin", + "model.layers.31.input_layernorm.weight": "pytorch_model-00004-of-00005.bin", + "model.layers.31.mlp.down_proj.weight": "pytorch_model-00004-of-00005.bin", + "model.layers.31.mlp.gate_proj.weight": "pytorch_model-00004-of-00005.bin", + "model.layers.31.mlp.up_proj.weight": "pytorch_model-00004-of-00005.bin", + "model.layers.31.post_attention_layernorm.weight": "pytorch_model-00004-of-00005.bin", + "model.layers.31.self_attn.k_proj.weight": "pytorch_model-00004-of-00005.bin", + "model.layers.31.self_attn.o_proj.weight": "pytorch_model-00004-of-00005.bin", + "model.layers.31.self_attn.q_proj.weight": "pytorch_model-00004-of-00005.bin", + "model.layers.31.self_attn.rotary_emb.inv_freq": "pytorch_model-00004-of-00005.bin", + "model.layers.31.self_attn.v_proj.weight": "pytorch_model-00004-of-00005.bin", + "model.layers.32.input_layernorm.weight": "pytorch_model-00004-of-00005.bin", + "model.layers.32.mlp.down_proj.weight": "pytorch_model-00004-of-00005.bin", + "model.layers.32.mlp.gate_proj.weight": "pytorch_model-00004-of-00005.bin", + "model.layers.32.mlp.up_proj.weight": "pytorch_model-00004-of-00005.bin", + "model.layers.32.post_attention_layernorm.weight": "pytorch_model-00004-of-00005.bin", + "model.layers.32.self_attn.k_proj.weight": "pytorch_model-00004-of-00005.bin", + "model.layers.32.self_attn.o_proj.weight": "pytorch_model-00004-of-00005.bin", + "model.layers.32.self_attn.q_proj.weight": "pytorch_model-00004-of-00005.bin", + "model.layers.32.self_attn.rotary_emb.inv_freq": "pytorch_model-00004-of-00005.bin", + "model.layers.32.self_attn.v_proj.weight": "pytorch_model-00004-of-00005.bin", + "model.layers.33.input_layernorm.weight": "pytorch_model-00004-of-00005.bin", + "model.layers.33.mlp.down_proj.weight": "pytorch_model-00004-of-00005.bin", + "model.layers.33.mlp.gate_proj.weight": "pytorch_model-00004-of-00005.bin", + "model.layers.33.mlp.up_proj.weight": "pytorch_model-00004-of-00005.bin", + "model.layers.33.post_attention_layernorm.weight": "pytorch_model-00004-of-00005.bin", + "model.layers.33.self_attn.k_proj.weight": "pytorch_model-00004-of-00005.bin", + "model.layers.33.self_attn.o_proj.weight": "pytorch_model-00004-of-00005.bin", + "model.layers.33.self_attn.q_proj.weight": "pytorch_model-00004-of-00005.bin", + "model.layers.33.self_attn.rotary_emb.inv_freq": "pytorch_model-00004-of-00005.bin", + "model.layers.33.self_attn.v_proj.weight": "pytorch_model-00004-of-00005.bin", + "model.layers.34.input_layernorm.weight": "pytorch_model-00004-of-00005.bin", + "model.layers.34.mlp.down_proj.weight": "pytorch_model-00004-of-00005.bin", + "model.layers.34.mlp.gate_proj.weight": "pytorch_model-00004-of-00005.bin", + "model.layers.34.mlp.up_proj.weight": "pytorch_model-00004-of-00005.bin", + "model.layers.34.post_attention_layernorm.weight": "pytorch_model-00004-of-00005.bin", + "model.layers.34.self_attn.k_proj.weight": "pytorch_model-00004-of-00005.bin", + "model.layers.34.self_attn.o_proj.weight": "pytorch_model-00004-of-00005.bin", + "model.layers.34.self_attn.q_proj.weight": "pytorch_model-00004-of-00005.bin", + "model.layers.34.self_attn.rotary_emb.inv_freq": "pytorch_model-00004-of-00005.bin", + "model.layers.34.self_attn.v_proj.weight": "pytorch_model-00004-of-00005.bin", + "model.layers.35.input_layernorm.weight": "pytorch_model-00004-of-00005.bin", + "model.layers.35.mlp.down_proj.weight": "pytorch_model-00004-of-00005.bin", + "model.layers.35.mlp.gate_proj.weight": "pytorch_model-00004-of-00005.bin", + "model.layers.35.mlp.up_proj.weight": "pytorch_model-00004-of-00005.bin", + "model.layers.35.post_attention_layernorm.weight": "pytorch_model-00004-of-00005.bin", + "model.layers.35.self_attn.k_proj.weight": "pytorch_model-00004-of-00005.bin", + "model.layers.35.self_attn.o_proj.weight": "pytorch_model-00004-of-00005.bin", + "model.layers.35.self_attn.q_proj.weight": "pytorch_model-00004-of-00005.bin", + "model.layers.35.self_attn.rotary_emb.inv_freq": "pytorch_model-00004-of-00005.bin", + "model.layers.35.self_attn.v_proj.weight": "pytorch_model-00004-of-00005.bin", + "model.layers.36.input_layernorm.weight": "pytorch_model-00005-of-00005.bin", + "model.layers.36.mlp.down_proj.weight": "pytorch_model-00005-of-00005.bin", + "model.layers.36.mlp.gate_proj.weight": "pytorch_model-00004-of-00005.bin", + "model.layers.36.mlp.up_proj.weight": "pytorch_model-00005-of-00005.bin", + "model.layers.36.post_attention_layernorm.weight": "pytorch_model-00005-of-00005.bin", + "model.layers.36.self_attn.k_proj.weight": "pytorch_model-00004-of-00005.bin", + "model.layers.36.self_attn.o_proj.weight": "pytorch_model-00004-of-00005.bin", + "model.layers.36.self_attn.q_proj.weight": "pytorch_model-00004-of-00005.bin", + "model.layers.36.self_attn.rotary_emb.inv_freq": "pytorch_model-00004-of-00005.bin", + "model.layers.36.self_attn.v_proj.weight": "pytorch_model-00004-of-00005.bin", + "model.layers.37.input_layernorm.weight": "pytorch_model-00005-of-00005.bin", + "model.layers.37.mlp.down_proj.weight": "pytorch_model-00005-of-00005.bin", + "model.layers.37.mlp.gate_proj.weight": "pytorch_model-00005-of-00005.bin", + "model.layers.37.mlp.up_proj.weight": "pytorch_model-00005-of-00005.bin", + "model.layers.37.post_attention_layernorm.weight": "pytorch_model-00005-of-00005.bin", + "model.layers.37.self_attn.k_proj.weight": "pytorch_model-00005-of-00005.bin", + "model.layers.37.self_attn.o_proj.weight": "pytorch_model-00005-of-00005.bin", + "model.layers.37.self_attn.q_proj.weight": "pytorch_model-00005-of-00005.bin", + "model.layers.37.self_attn.rotary_emb.inv_freq": "pytorch_model-00005-of-00005.bin", + "model.layers.37.self_attn.v_proj.weight": "pytorch_model-00005-of-00005.bin", + "model.layers.38.input_layernorm.weight": "pytorch_model-00005-of-00005.bin", + "model.layers.38.mlp.down_proj.weight": "pytorch_model-00005-of-00005.bin", + "model.layers.38.mlp.gate_proj.weight": "pytorch_model-00005-of-00005.bin", + "model.layers.38.mlp.up_proj.weight": "pytorch_model-00005-of-00005.bin", + "model.layers.38.post_attention_layernorm.weight": "pytorch_model-00005-of-00005.bin", + "model.layers.38.self_attn.k_proj.weight": "pytorch_model-00005-of-00005.bin", + "model.layers.38.self_attn.o_proj.weight": "pytorch_model-00005-of-00005.bin", + "model.layers.38.self_attn.q_proj.weight": "pytorch_model-00005-of-00005.bin", + "model.layers.38.self_attn.rotary_emb.inv_freq": "pytorch_model-00005-of-00005.bin", + "model.layers.38.self_attn.v_proj.weight": "pytorch_model-00005-of-00005.bin", + "model.layers.39.input_layernorm.weight": "pytorch_model-00005-of-00005.bin", + "model.layers.39.mlp.down_proj.weight": "pytorch_model-00005-of-00005.bin", + "model.layers.39.mlp.gate_proj.weight": "pytorch_model-00005-of-00005.bin", + "model.layers.39.mlp.up_proj.weight": "pytorch_model-00005-of-00005.bin", + "model.layers.39.post_attention_layernorm.weight": "pytorch_model-00005-of-00005.bin", + "model.layers.39.self_attn.k_proj.weight": "pytorch_model-00005-of-00005.bin", + "model.layers.39.self_attn.o_proj.weight": "pytorch_model-00005-of-00005.bin", + "model.layers.39.self_attn.q_proj.weight": "pytorch_model-00005-of-00005.bin", + "model.layers.39.self_attn.rotary_emb.inv_freq": "pytorch_model-00005-of-00005.bin", + "model.layers.39.self_attn.v_proj.weight": "pytorch_model-00005-of-00005.bin", + "model.layers.4.input_layernorm.weight": "pytorch_model-00001-of-00005.bin", + "model.layers.4.mlp.down_proj.weight": "pytorch_model-00001-of-00005.bin", + "model.layers.4.mlp.gate_proj.weight": "pytorch_model-00001-of-00005.bin", + "model.layers.4.mlp.up_proj.weight": "pytorch_model-00001-of-00005.bin", + "model.layers.4.post_attention_layernorm.weight": "pytorch_model-00001-of-00005.bin", + "model.layers.4.self_attn.k_proj.weight": "pytorch_model-00001-of-00005.bin", + "model.layers.4.self_attn.o_proj.weight": "pytorch_model-00001-of-00005.bin", + "model.layers.4.self_attn.q_proj.weight": "pytorch_model-00001-of-00005.bin", + "model.layers.4.self_attn.rotary_emb.inv_freq": "pytorch_model-00001-of-00005.bin", + "model.layers.4.self_attn.v_proj.weight": "pytorch_model-00001-of-00005.bin", + "model.layers.5.input_layernorm.weight": "pytorch_model-00001-of-00005.bin", + "model.layers.5.mlp.down_proj.weight": "pytorch_model-00001-of-00005.bin", + "model.layers.5.mlp.gate_proj.weight": "pytorch_model-00001-of-00005.bin", + "model.layers.5.mlp.up_proj.weight": "pytorch_model-00001-of-00005.bin", + "model.layers.5.post_attention_layernorm.weight": "pytorch_model-00001-of-00005.bin", + "model.layers.5.self_attn.k_proj.weight": "pytorch_model-00001-of-00005.bin", + "model.layers.5.self_attn.o_proj.weight": "pytorch_model-00001-of-00005.bin", + "model.layers.5.self_attn.q_proj.weight": "pytorch_model-00001-of-00005.bin", + "model.layers.5.self_attn.rotary_emb.inv_freq": "pytorch_model-00001-of-00005.bin", + "model.layers.5.self_attn.v_proj.weight": "pytorch_model-00001-of-00005.bin", + "model.layers.6.input_layernorm.weight": "pytorch_model-00001-of-00005.bin", + "model.layers.6.mlp.down_proj.weight": "pytorch_model-00001-of-00005.bin", + "model.layers.6.mlp.gate_proj.weight": "pytorch_model-00001-of-00005.bin", + "model.layers.6.mlp.up_proj.weight": "pytorch_model-00001-of-00005.bin", + "model.layers.6.post_attention_layernorm.weight": "pytorch_model-00001-of-00005.bin", + "model.layers.6.self_attn.k_proj.weight": "pytorch_model-00001-of-00005.bin", + "model.layers.6.self_attn.o_proj.weight": "pytorch_model-00001-of-00005.bin", + "model.layers.6.self_attn.q_proj.weight": "pytorch_model-00001-of-00005.bin", + "model.layers.6.self_attn.rotary_emb.inv_freq": "pytorch_model-00001-of-00005.bin", + "model.layers.6.self_attn.v_proj.weight": "pytorch_model-00001-of-00005.bin", + "model.layers.7.input_layernorm.weight": "pytorch_model-00001-of-00005.bin", + "model.layers.7.mlp.down_proj.weight": "pytorch_model-00001-of-00005.bin", + "model.layers.7.mlp.gate_proj.weight": "pytorch_model-00001-of-00005.bin", + "model.layers.7.mlp.up_proj.weight": "pytorch_model-00001-of-00005.bin", + "model.layers.7.post_attention_layernorm.weight": "pytorch_model-00001-of-00005.bin", + "model.layers.7.self_attn.k_proj.weight": "pytorch_model-00001-of-00005.bin", + "model.layers.7.self_attn.o_proj.weight": "pytorch_model-00001-of-00005.bin", + "model.layers.7.self_attn.q_proj.weight": "pytorch_model-00001-of-00005.bin", + "model.layers.7.self_attn.rotary_emb.inv_freq": "pytorch_model-00001-of-00005.bin", + "model.layers.7.self_attn.v_proj.weight": "pytorch_model-00001-of-00005.bin", + "model.layers.8.input_layernorm.weight": "pytorch_model-00002-of-00005.bin", + "model.layers.8.mlp.down_proj.weight": "pytorch_model-00002-of-00005.bin", + "model.layers.8.mlp.gate_proj.weight": "pytorch_model-00001-of-00005.bin", + "model.layers.8.mlp.up_proj.weight": "pytorch_model-00001-of-00005.bin", + "model.layers.8.post_attention_layernorm.weight": "pytorch_model-00002-of-00005.bin", + "model.layers.8.self_attn.k_proj.weight": "pytorch_model-00001-of-00005.bin", + "model.layers.8.self_attn.o_proj.weight": "pytorch_model-00001-of-00005.bin", + "model.layers.8.self_attn.q_proj.weight": "pytorch_model-00001-of-00005.bin", + "model.layers.8.self_attn.rotary_emb.inv_freq": "pytorch_model-00001-of-00005.bin", + "model.layers.8.self_attn.v_proj.weight": "pytorch_model-00001-of-00005.bin", + "model.layers.9.input_layernorm.weight": "pytorch_model-00002-of-00005.bin", + "model.layers.9.mlp.down_proj.weight": "pytorch_model-00002-of-00005.bin", + "model.layers.9.mlp.gate_proj.weight": "pytorch_model-00002-of-00005.bin", + "model.layers.9.mlp.up_proj.weight": "pytorch_model-00002-of-00005.bin", + "model.layers.9.post_attention_layernorm.weight": "pytorch_model-00002-of-00005.bin", + "model.layers.9.self_attn.k_proj.weight": "pytorch_model-00002-of-00005.bin", + "model.layers.9.self_attn.o_proj.weight": "pytorch_model-00002-of-00005.bin", + "model.layers.9.self_attn.q_proj.weight": "pytorch_model-00002-of-00005.bin", + "model.layers.9.self_attn.rotary_emb.inv_freq": "pytorch_model-00002-of-00005.bin", + "model.layers.9.self_attn.v_proj.weight": "pytorch_model-00002-of-00005.bin", + "model.norm.weight": "pytorch_model-00005-of-00005.bin" + } +} diff --git a/special_tokens_map.json b/special_tokens_map.json new file mode 100644 index 0000000..f928b24 --- /dev/null +++ b/special_tokens_map.json @@ -0,0 +1,24 @@ +{ + "bos_token": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "pad_token": "", + "unk_token": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false + } +} diff --git a/tokenizer.model b/tokenizer.model new file mode 100644 index 0000000..6c00c74 --- /dev/null +++ b/tokenizer.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347 +size 499723 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..a9608c1 --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,34 @@ +{ + "add_bos_token": true, + "add_eos_token": false, + "bos_token": { + "__type": "AddedToken", + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "clean_up_tokenization_spaces": false, + "eos_token": { + "__type": "AddedToken", + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "legacy": false, + "model_max_length": 1000000000000000019884624838656, + "pad_token": null, + "sp_model_kwargs": {}, + "tokenizer_class": "LlamaTokenizer", + "unk_token": { + "__type": "AddedToken", + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false + } +}