chargoddard/llama2-22b
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

初始化项目，由ModelHub XC社区提供模型

Browse Source 
Model: chargoddard/llama2-22b
Source: Original Platform
This commit is contained in:
ModelHub XC
2026-06-28 22:21:21 +08:00
commit be0b8de887
14 changed files with 769 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
.gitattributes vendored Normal file
View File

@@ -0,0 +1,35 @@
*.7z filter=lfs diff=lfs merge=lfs -text
*.arrow filter=lfs diff=lfs merge=lfs -text
*.bin filter=lfs diff=lfs merge=lfs -text
*.bz2 filter=lfs diff=lfs merge=lfs -text
*.ckpt filter=lfs diff=lfs merge=lfs -text
*.ftz filter=lfs diff=lfs merge=lfs -text
*.gz filter=lfs diff=lfs merge=lfs -text
*.h5 filter=lfs diff=lfs merge=lfs -text
*.joblib filter=lfs diff=lfs merge=lfs -text
*.lfs.* filter=lfs diff=lfs merge=lfs -text
*.mlmodel filter=lfs diff=lfs merge=lfs -text
*.model filter=lfs diff=lfs merge=lfs -text
*.msgpack filter=lfs diff=lfs merge=lfs -text
*.npy filter=lfs diff=lfs merge=lfs -text
*.npz filter=lfs diff=lfs merge=lfs -text
*.onnx filter=lfs diff=lfs merge=lfs -text
*.ot filter=lfs diff=lfs merge=lfs -text
*.parquet filter=lfs diff=lfs merge=lfs -text
*.pb filter=lfs diff=lfs merge=lfs -text
*.pickle filter=lfs diff=lfs merge=lfs -text
*.pkl filter=lfs diff=lfs merge=lfs -text
*.pt filter=lfs diff=lfs merge=lfs -text
*.pth filter=lfs diff=lfs merge=lfs -text
*.rar filter=lfs diff=lfs merge=lfs -text
*.safetensors filter=lfs diff=lfs merge=lfs -text
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
*.tar.* filter=lfs diff=lfs merge=lfs -text
*.tar filter=lfs diff=lfs merge=lfs -text
*.tflite filter=lfs diff=lfs merge=lfs -text
*.tgz filter=lfs diff=lfs merge=lfs -text
*.wasm filter=lfs diff=lfs merge=lfs -text
*.xz filter=lfs diff=lfs merge=lfs -text
*.zip filter=lfs diff=lfs merge=lfs -text
*.zst filter=lfs diff=lfs merge=lfs -text
*tfevents* filter=lfs diff=lfs merge=lfs -text

27
README.md Normal file
View File

@@ -0,0 +1,27 @@
---
model_type: llama
pipeline_tag: text-generation
datasets:
- togethercomputer/RedPajama-Data-1T-Sample
tags:
- llama
---
This is [Llama 2 13b](https://huggingface.co/meta-llama/Llama-2-13b-hf) with some additional attention heads from original-flavor Llama 33b frankensteined on.
Fine-tuned on ~10M tokens from RedPajama to settle in the transplants a little.
Not intended for use as-is - this model is meant to serve as a base for further tuning, hopefully with a greater capacity for learning than 13b.
# [Open LLM Leaderboard Evaluation Results](https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard)
Detailed results can be found [here](https://huggingface.co/datasets/open-llm-leaderboard/details_chargoddard__llama2-22b)
| Metric | Value |
|-----------------------|---------------------------|
| Avg. | 46.85 |
| ARC (25-shot) | 58.53 |
| HellaSwag (10-shot) | 82.55 |
| MMLU (5-shot) | 54.68 |
| TruthfulQA (0-shot) | 39.84 |
| Winogrande (5-shot) | 76.32 |
| GSM8K (5-shot) | 9.93 |
| DROP (3-shot) | 6.08 |

26
config.json Normal file
View File

@@ -0,0 +1,26 @@
{
"_name_or_path": "/root/llama2-22b/",
"architectures": [
"LlamaForCausalLM"
],
"bos_token_id": 1,
"eos_token_id": 2,
"hidden_act": "silu",
"hidden_size": 6656,
"initializer_range": 0.02,
"intermediate_size": 17920,
"max_position_embeddings": 4098,
"model_type": "llama",
"num_attention_heads": 52,
"num_hidden_layers": 40,
"num_key_value_heads": 52,
"pad_token_id": 0,
"pretraining_tp": 1,
"rms_norm_eps": 1e-05,
"rope_scaling": null,
"tie_word_embeddings": false,
"torch_dtype": "float16",
"transformers_version": "4.32.0.dev0",
"use_cache": false,
"vocab_size": 32000
}

188
frankenllama_22b.py Normal file
View File

@@ -0,0 +1,188 @@
#!/usr/bin/env python3
# Charles O. Goddard
# 7/20/2023
"""Script used to generate the base frankenmerge. Output will need fine-tuning to be useful."""
import copy
import torch
from torch import Tensor, nn
import transformers
from transformers.models.llama.modeling_llama import (
LlamaForCausalLM,
LlamaDecoderLayer,
)
from transformers import LlamaForCausalLM, LlamaConfig
import torch
import transformers
import numpy as np
MODEL_NAME_13B = "meta-llama/Llama-2-13b-hf" # primary model
MODEL_NAME_33B = "huggyllama/llama-30b" # donor
BLOCK_DIAGONAL = True
# If BLOCK_DIAGONAL is set to True, each tensor in the resultant model will form a
# block diagonal matrix, as illustrated below:
# a a a 0 0
# a a a 0 0
# a a a 0 0
# 0 0 0 b b
# 0 0 0 b b
# In this configuration, the states (hidden and intermediate) from the original
# and donor models are completely decoupled. That is, the hidden states
# corresponding to the original model remain unchanged, and the new dimensions
# added from the donor model do not depend on the hidden states of the original model.
# If BLOCK_DIAGONAL is set to False, the tensors will instead have the following form:
# a a a 0 0
# a a a 0 0
# a a a 0 0
# b b b b b
# b b b b b
# In this case, the output of the newly added attention heads depends on the hidden
# state values as if they were part of the donor model. Although the original model's
# hidden states remain unchanged in either case, interaction between the new and old
# features will result in features of varying usefulness.
class NoInit:
def __enter__(self):
def noop(*args, **kwargs):
pass
(k, u, n) = (
torch.nn.init.kaiming_uniform_,
torch.nn.init.uniform_,
torch.nn.init.normal_,
)
torch.nn.init.kaiming_uniform_ = noop
torch.nn.init.uniform_ = noop
torch.nn.init.normal_ = noop
transformers.modeling_utils._init_weights = False
self.funcs = (k, u, n)
def __exit__(self, *args):
(k, u, n) = self.funcs
(
torch.nn.init.kaiming_uniform_,
torch.nn.init.uniform_,
torch.nn.init.normal_,
) = (
k,
u,
n,
)
transformers.modeling_utils._init_weights = True
def format_kmb(n, digits=None):
n = int(n)
if n < 1000:
return str(n)
elif n < 1000_000:
return f"{round(n/1000, digits)}k"
elif n < 1000 * 1000 * 1000:
return f"{round(n/(1000*1000), digits)}m"
else:
return f"{round(n/(1000*1000*1000), digits)}b"
def count_params(model):
model_parameters = filter(lambda p: p.requires_grad, model.parameters())
params = sum([np.prod(p.size()) for p in model_parameters])
return int(params)
torch.set_default_dtype(torch.float16)
config_13b: LlamaConfig = LlamaConfig.from_pretrained(MODEL_NAME_13B)
config_33b: LlamaConfig = LlamaConfig.from_pretrained(MODEL_NAME_33B)
config_more = copy.deepcopy(config_13b)
config_more.intermediate_size = config_33b.intermediate_size
config_more.hidden_size = config_33b.hidden_size
config_more.num_key_value_heads = config_33b.num_key_value_heads
config_more.num_attention_heads = config_33b.num_key_value_heads
print(config_more)
with NoInit():
model = LlamaForCausalLM(config_more)
print(f"{format_kmb(count_params(model), 3)} parameters")
def merge_tensors_inplace(dest: Tensor, s0: Tensor, s1: Tensor, block_diagonal: bool):
dest.zero_()
if block_diagonal:
dest[s0.shape[0] :, s0.shape[1] :] = s1[
s0.shape[0] : dest.shape[0],
s0.shape[1] : dest.shape[1],
]
else:
dest[s0.shape[0] :, :] = s1[
s0.shape[0] : dest.shape[0],
: dest.shape[1],
]
dest[: s0.shape[0], : s0.shape[1]] = s0
with NoInit():
donor_13b = (
LlamaForCausalLM.from_pretrained(MODEL_NAME_13B).to(torch.float16).eval()
)
donor_33b = (
LlamaForCausalLM.from_pretrained(MODEL_NAME_33B).to(torch.float16).eval()
)
with torch.no_grad():
for layer_idx in range(len(model.model.layers)):
layer: LlamaDecoderLayer = model.model.layers[layer_idx]
l13: LlamaDecoderLayer = donor_13b.model.layers[layer_idx]
l33: LlamaDecoderLayer = donor_33b.model.layers[layer_idx]
for name in ("q_proj", "k_proj", "v_proj", "o_proj"):
dest: nn.Linear = getattr(layer.self_attn, name)
s13: nn.Linear = getattr(l13.self_attn, name)
s33: nn.Linear = getattr(l33.self_attn, name)
merge_tensors_inplace(dest.weight, s13.weight, s33.weight, BLOCK_DIAGONAL)
for name in ("up_proj", "gate_proj", "down_proj"):
dest: nn.Linear = getattr(layer.mlp, name)
s13: nn.Linear = getattr(l13.mlp, name)
s33: nn.Linear = getattr(l33.mlp, name)
merge_tensors_inplace(dest.weight, s13.weight, s33.weight, BLOCK_DIAGONAL)
layer.input_layernorm.weight[:] = l33.input_layernorm.weight[
: layer.input_layernorm.weight.shape[0]
]
layer.input_layernorm.weight[
: l13.input_layernorm.weight.shape[0]
] = l13.input_layernorm.weight
layer.post_attention_layernorm.weight[:] = l33.post_attention_layernorm.weight[
: layer.post_attention_layernorm.weight.shape[0]
]
layer.post_attention_layernorm.weight[
: l13.post_attention_layernorm.weight.shape[0]
] = l13.post_attention_layernorm.weight
# have initial output depend on only original llama2-13b features, so model
# starts unimpaired and can learn to incorporate the new features as well
model.lm_head.weight.zero_()
model.lm_head.weight[
: donor_13b.lm_head.weight.shape[0], : donor_13b.lm_head.weight.shape[1]
] = donor_13b.lm_head.weight
merge_tensors_inplace(
model.model.embed_tokens.weight,
donor_13b.model.embed_tokens.weight,
donor_33b.model.embed_tokens.weight,
BLOCK_DIAGONAL,
)
model.save_pretrained("./llama2-22b/", safe_serialization=True)

7
generation_config.json Normal file
View File

@@ -0,0 +1,7 @@
{
"_from_model_config": true,
"bos_token_id": 1,
"eos_token_id": 2,
"pad_token_id": 0,
"transformers_version": "4.32.0.dev0"
}

3
pytorch_model-00001-of-00005.bin Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:70019781140edfa10f01730a1e20b649a82cd104981bbd42eb175c985c581741
size 9818324691

3
pytorch_model-00002-of-00005.bin Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:8d96b90f05e26d973b86c512c33b121683c30efd88eb6ee2bca08fae7a128366
size 9958102871

3
pytorch_model-00003-of-00005.bin Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:dd2b8673c387c5497edbeee036c9fb6072f0101deb7fb97158fc53cb65595ed0
size 9896734715

3
pytorch_model-00004-of-00005.bin Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:b60d983a87dd4f8450583eedda634dc452152a719ea6ce86bbd77548dd82240d
size 9869470609

3
pytorch_model-00005-of-00005.bin Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:d950dcd51ecfdc5cc77c357acb2dd2766cbda8c538ab6e333f62abef67feac0c
size 4113434135

410
pytorch_model.bin.index.json Normal file
View File

@@ -0,0 +1,410 @@
{
"metadata": {
"total_size": 43655923712
},
"weight_map": {
"lm_head.weight": "pytorch_model-00005-of-00005.bin",
"model.embed_tokens.weight": "pytorch_model-00001-of-00005.bin",
"model.layers.0.input_layernorm.weight": "pytorch_model-00001-of-00005.bin",
"model.layers.0.mlp.down_proj.weight": "pytorch_model-00001-of-00005.bin",
"model.layers.0.mlp.gate_proj.weight": "pytorch_model-00001-of-00005.bin",
"model.layers.0.mlp.up_proj.weight": "pytorch_model-00001-of-00005.bin",
"model.layers.0.post_attention_layernorm.weight": "pytorch_model-00001-of-00005.bin",
"model.layers.0.self_attn.k_proj.weight": "pytorch_model-00001-of-00005.bin",
"model.layers.0.self_attn.o_proj.weight": "pytorch_model-00001-of-00005.bin",
"model.layers.0.self_attn.q_proj.weight": "pytorch_model-00001-of-00005.bin",
"model.layers.0.self_attn.rotary_emb.inv_freq": "pytorch_model-00001-of-00005.bin",
"model.layers.0.self_attn.v_proj.weight": "pytorch_model-00001-of-00005.bin",
"model.layers.1.input_layernorm.weight": "pytorch_model-00001-of-00005.bin",
"model.layers.1.mlp.down_proj.weight": "pytorch_model-00001-of-00005.bin",
"model.layers.1.mlp.gate_proj.weight": "pytorch_model-00001-of-00005.bin",
"model.layers.1.mlp.up_proj.weight": "pytorch_model-00001-of-00005.bin",
"model.layers.1.post_attention_layernorm.weight": "pytorch_model-00001-of-00005.bin",
"model.layers.1.self_attn.k_proj.weight": "pytorch_model-00001-of-00005.bin",
"model.layers.1.self_attn.o_proj.weight": "pytorch_model-00001-of-00005.bin",
"model.layers.1.self_attn.q_proj.weight": "pytorch_model-00001-of-00005.bin",
"model.layers.1.self_attn.rotary_emb.inv_freq": "pytorch_model-00001-of-00005.bin",
"model.layers.1.self_attn.v_proj.weight": "pytorch_model-00001-of-00005.bin",
"model.layers.10.input_layernorm.weight": "pytorch_model-00002-of-00005.bin",
"model.layers.10.mlp.down_proj.weight": "pytorch_model-00002-of-00005.bin",
"model.layers.10.mlp.gate_proj.weight": "pytorch_model-00002-of-00005.bin",
"model.layers.10.mlp.up_proj.weight": "pytorch_model-00002-of-00005.bin",
"model.layers.10.post_attention_layernorm.weight": "pytorch_model-00002-of-00005.bin",
"model.layers.10.self_attn.k_proj.weight": "pytorch_model-00002-of-00005.bin",
"model.layers.10.self_attn.o_proj.weight": "pytorch_model-00002-of-00005.bin",
"model.layers.10.self_attn.q_proj.weight": "pytorch_model-00002-of-00005.bin",
"model.layers.10.self_attn.rotary_emb.inv_freq": "pytorch_model-00002-of-00005.bin",
"model.layers.10.self_attn.v_proj.weight": "pytorch_model-00002-of-00005.bin",
"model.layers.11.input_layernorm.weight": "pytorch_model-00002-of-00005.bin",
"model.layers.11.mlp.down_proj.weight": "pytorch_model-00002-of-00005.bin",
"model.layers.11.mlp.gate_proj.weight": "pytorch_model-00002-of-00005.bin",
"model.layers.11.mlp.up_proj.weight": "pytorch_model-00002-of-00005.bin",
"model.layers.11.post_attention_layernorm.weight": "pytorch_model-00002-of-00005.bin",
"model.layers.11.self_attn.k_proj.weight": "pytorch_model-00002-of-00005.bin",
"model.layers.11.self_attn.o_proj.weight": "pytorch_model-00002-of-00005.bin",
"model.layers.11.self_attn.q_proj.weight": "pytorch_model-00002-of-00005.bin",
"model.layers.11.self_attn.rotary_emb.inv_freq": "pytorch_model-00002-of-00005.bin",
"model.layers.11.self_attn.v_proj.weight": "pytorch_model-00002-of-00005.bin",
"model.layers.12.input_layernorm.weight": "pytorch_model-00002-of-00005.bin",
"model.layers.12.mlp.down_proj.weight": "pytorch_model-00002-of-00005.bin",
"model.layers.12.mlp.gate_proj.weight": "pytorch_model-00002-of-00005.bin",
"model.layers.12.mlp.up_proj.weight": "pytorch_model-00002-of-00005.bin",
"model.layers.12.post_attention_layernorm.weight": "pytorch_model-00002-of-00005.bin",
"model.layers.12.self_attn.k_proj.weight": "pytorch_model-00002-of-00005.bin",
"model.layers.12.self_attn.o_proj.weight": "pytorch_model-00002-of-00005.bin",
"model.layers.12.self_attn.q_proj.weight": "pytorch_model-00002-of-00005.bin",
"model.layers.12.self_attn.rotary_emb.inv_freq": "pytorch_model-00002-of-00005.bin",
"model.layers.12.self_attn.v_proj.weight": "pytorch_model-00002-of-00005.bin",
"model.layers.13.input_layernorm.weight": "pytorch_model-00002-of-00005.bin",
"model.layers.13.mlp.down_proj.weight": "pytorch_model-00002-of-00005.bin",
"model.layers.13.mlp.gate_proj.weight": "pytorch_model-00002-of-00005.bin",
"model.layers.13.mlp.up_proj.weight": "pytorch_model-00002-of-00005.bin",
"model.layers.13.post_attention_layernorm.weight": "pytorch_model-00002-of-00005.bin",
"model.layers.13.self_attn.k_proj.weight": "pytorch_model-00002-of-00005.bin",
"model.layers.13.self_attn.o_proj.weight": "pytorch_model-00002-of-00005.bin",
"model.layers.13.self_attn.q_proj.weight": "pytorch_model-00002-of-00005.bin",
"model.layers.13.self_attn.rotary_emb.inv_freq": "pytorch_model-00002-of-00005.bin",
"model.layers.13.self_attn.v_proj.weight": "pytorch_model-00002-of-00005.bin",
"model.layers.14.input_layernorm.weight": "pytorch_model-00002-of-00005.bin",
"model.layers.14.mlp.down_proj.weight": "pytorch_model-00002-of-00005.bin",
"model.layers.14.mlp.gate_proj.weight": "pytorch_model-00002-of-00005.bin",
"model.layers.14.mlp.up_proj.weight": "pytorch_model-00002-of-00005.bin",
"model.layers.14.post_attention_layernorm.weight": "pytorch_model-00002-of-00005.bin",
"model.layers.14.self_attn.k_proj.weight": "pytorch_model-00002-of-00005.bin",
"model.layers.14.self_attn.o_proj.weight": "pytorch_model-00002-of-00005.bin",
"model.layers.14.self_attn.q_proj.weight": "pytorch_model-00002-of-00005.bin",
"model.layers.14.self_attn.rotary_emb.inv_freq": "pytorch_model-00002-of-00005.bin",
"model.layers.14.self_attn.v_proj.weight": "pytorch_model-00002-of-00005.bin",
"model.layers.15.input_layernorm.weight": "pytorch_model-00002-of-00005.bin",
"model.layers.15.mlp.down_proj.weight": "pytorch_model-00002-of-00005.bin",
"model.layers.15.mlp.gate_proj.weight": "pytorch_model-00002-of-00005.bin",
"model.layers.15.mlp.up_proj.weight": "pytorch_model-00002-of-00005.bin",
"model.layers.15.post_attention_layernorm.weight": "pytorch_model-00002-of-00005.bin",
"model.layers.15.self_attn.k_proj.weight": "pytorch_model-00002-of-00005.bin",
"model.layers.15.self_attn.o_proj.weight": "pytorch_model-00002-of-00005.bin",
"model.layers.15.self_attn.q_proj.weight": "pytorch_model-00002-of-00005.bin",
"model.layers.15.self_attn.rotary_emb.inv_freq": "pytorch_model-00002-of-00005.bin",
"model.layers.15.self_attn.v_proj.weight": "pytorch_model-00002-of-00005.bin",
"model.layers.16.input_layernorm.weight": "pytorch_model-00002-of-00005.bin",
"model.layers.16.mlp.down_proj.weight": "pytorch_model-00002-of-00005.bin",
"model.layers.16.mlp.gate_proj.weight": "pytorch_model-00002-of-00005.bin",
"model.layers.16.mlp.up_proj.weight": "pytorch_model-00002-of-00005.bin",
"model.layers.16.post_attention_layernorm.weight": "pytorch_model-00002-of-00005.bin",
"model.layers.16.self_attn.k_proj.weight": "pytorch_model-00002-of-00005.bin",
"model.layers.16.self_attn.o_proj.weight": "pytorch_model-00002-of-00005.bin",
"model.layers.16.self_attn.q_proj.weight": "pytorch_model-00002-of-00005.bin",
"model.layers.16.self_attn.rotary_emb.inv_freq": "pytorch_model-00002-of-00005.bin",
"model.layers.16.self_attn.v_proj.weight": "pytorch_model-00002-of-00005.bin",
"model.layers.17.input_layernorm.weight": "pytorch_model-00002-of-00005.bin",
"model.layers.17.mlp.down_proj.weight": "pytorch_model-00002-of-00005.bin",
"model.layers.17.mlp.gate_proj.weight": "pytorch_model-00002-of-00005.bin",
"model.layers.17.mlp.up_proj.weight": "pytorch_model-00002-of-00005.bin",
"model.layers.17.post_attention_layernorm.weight": "pytorch_model-00002-of-00005.bin",
"model.layers.17.self_attn.k_proj.weight": "pytorch_model-00002-of-00005.bin",
"model.layers.17.self_attn.o_proj.weight": "pytorch_model-00002-of-00005.bin",
"model.layers.17.self_attn.q_proj.weight": "pytorch_model-00002-of-00005.bin",
"model.layers.17.self_attn.rotary_emb.inv_freq": "pytorch_model-00002-of-00005.bin",
"model.layers.17.self_attn.v_proj.weight": "pytorch_model-00002-of-00005.bin",
"model.layers.18.input_layernorm.weight": "pytorch_model-00003-of-00005.bin",
"model.layers.18.mlp.down_proj.weight": "pytorch_model-00003-of-00005.bin",
"model.layers.18.mlp.gate_proj.weight": "pytorch_model-00003-of-00005.bin",
"model.layers.18.mlp.up_proj.weight": "pytorch_model-00003-of-00005.bin",
"model.layers.18.post_attention_layernorm.weight": "pytorch_model-00003-of-00005.bin",
"model.layers.18.self_attn.k_proj.weight": "pytorch_model-00003-of-00005.bin",
"model.layers.18.self_attn.o_proj.weight": "pytorch_model-00003-of-00005.bin",
"model.layers.18.self_attn.q_proj.weight": "pytorch_model-00002-of-00005.bin",
"model.layers.18.self_attn.rotary_emb.inv_freq": "pytorch_model-00003-of-00005.bin",
"model.layers.18.self_attn.v_proj.weight": "pytorch_model-00003-of-00005.bin",
"model.layers.19.input_layernorm.weight": "pytorch_model-00003-of-00005.bin",
"model.layers.19.mlp.down_proj.weight": "pytorch_model-00003-of-00005.bin",
"model.layers.19.mlp.gate_proj.weight": "pytorch_model-00003-of-00005.bin",
"model.layers.19.mlp.up_proj.weight": "pytorch_model-00003-of-00005.bin",
"model.layers.19.post_attention_layernorm.weight": "pytorch_model-00003-of-00005.bin",
"model.layers.19.self_attn.k_proj.weight": "pytorch_model-00003-of-00005.bin",
"model.layers.19.self_attn.o_proj.weight": "pytorch_model-00003-of-00005.bin",
"model.layers.19.self_attn.q_proj.weight": "pytorch_model-00003-of-00005.bin",
"model.layers.19.self_attn.rotary_emb.inv_freq": "pytorch_model-00003-of-00005.bin",
"model.layers.19.self_attn.v_proj.weight": "pytorch_model-00003-of-00005.bin",
"model.layers.2.input_layernorm.weight": "pytorch_model-00001-of-00005.bin",
"model.layers.2.mlp.down_proj.weight": "pytorch_model-00001-of-00005.bin",
"model.layers.2.mlp.gate_proj.weight": "pytorch_model-00001-of-00005.bin",
"model.layers.2.mlp.up_proj.weight": "pytorch_model-00001-of-00005.bin",
"model.layers.2.post_attention_layernorm.weight": "pytorch_model-00001-of-00005.bin",
"model.layers.2.self_attn.k_proj.weight": "pytorch_model-00001-of-00005.bin",
"model.layers.2.self_attn.o_proj.weight": "pytorch_model-00001-of-00005.bin",
"model.layers.2.self_attn.q_proj.weight": "pytorch_model-00001-of-00005.bin",
"model.layers.2.self_attn.rotary_emb.inv_freq": "pytorch_model-00001-of-00005.bin",
"model.layers.2.self_attn.v_proj.weight": "pytorch_model-00001-of-00005.bin",
"model.layers.20.input_layernorm.weight": "pytorch_model-00003-of-00005.bin",
"model.layers.20.mlp.down_proj.weight": "pytorch_model-00003-of-00005.bin",
"model.layers.20.mlp.gate_proj.weight": "pytorch_model-00003-of-00005.bin",
"model.layers.20.mlp.up_proj.weight": "pytorch_model-00003-of-00005.bin",
"model.layers.20.post_attention_layernorm.weight": "pytorch_model-00003-of-00005.bin",
"model.layers.20.self_attn.k_proj.weight": "pytorch_model-00003-of-00005.bin",
"model.layers.20.self_attn.o_proj.weight": "pytorch_model-00003-of-00005.bin",
"model.layers.20.self_attn.q_proj.weight": "pytorch_model-00003-of-00005.bin",
"model.layers.20.self_attn.rotary_emb.inv_freq": "pytorch_model-00003-of-00005.bin",
"model.layers.20.self_attn.v_proj.weight": "pytorch_model-00003-of-00005.bin",
"model.layers.21.input_layernorm.weight": "pytorch_model-00003-of-00005.bin",
"model.layers.21.mlp.down_proj.weight": "pytorch_model-00003-of-00005.bin",
"model.layers.21.mlp.gate_proj.weight": "pytorch_model-00003-of-00005.bin",
"model.layers.21.mlp.up_proj.weight": "pytorch_model-00003-of-00005.bin",
"model.layers.21.post_attention_layernorm.weight": "pytorch_model-00003-of-00005.bin",
"model.layers.21.self_attn.k_proj.weight": "pytorch_model-00003-of-00005.bin",
"model.layers.21.self_attn.o_proj.weight": "pytorch_model-00003-of-00005.bin",
"model.layers.21.self_attn.q_proj.weight": "pytorch_model-00003-of-00005.bin",
"model.layers.21.self_attn.rotary_emb.inv_freq": "pytorch_model-00003-of-00005.bin",
"model.layers.21.self_attn.v_proj.weight": "pytorch_model-00003-of-00005.bin",
"model.layers.22.input_layernorm.weight": "pytorch_model-00003-of-00005.bin",
"model.layers.22.mlp.down_proj.weight": "pytorch_model-00003-of-00005.bin",
"model.layers.22.mlp.gate_proj.weight": "pytorch_model-00003-of-00005.bin",
"model.layers.22.mlp.up_proj.weight": "pytorch_model-00003-of-00005.bin",
"model.layers.22.post_attention_layernorm.weight": "pytorch_model-00003-of-00005.bin",
"model.layers.22.self_attn.k_proj.weight": "pytorch_model-00003-of-00005.bin",
"model.layers.22.self_attn.o_proj.weight": "pytorch_model-00003-of-00005.bin",
"model.layers.22.self_attn.q_proj.weight": "pytorch_model-00003-of-00005.bin",
"model.layers.22.self_attn.rotary_emb.inv_freq": "pytorch_model-00003-of-00005.bin",
"model.layers.22.self_attn.v_proj.weight": "pytorch_model-00003-of-00005.bin",
"model.layers.23.input_layernorm.weight": "pytorch_model-00003-of-00005.bin",
"model.layers.23.mlp.down_proj.weight": "pytorch_model-00003-of-00005.bin",
"model.layers.23.mlp.gate_proj.weight": "pytorch_model-00003-of-00005.bin",
"model.layers.23.mlp.up_proj.weight": "pytorch_model-00003-of-00005.bin",
"model.layers.23.post_attention_layernorm.weight": "pytorch_model-00003-of-00005.bin",
"model.layers.23.self_attn.k_proj.weight": "pytorch_model-00003-of-00005.bin",
"model.layers.23.self_attn.o_proj.weight": "pytorch_model-00003-of-00005.bin",
"model.layers.23.self_attn.q_proj.weight": "pytorch_model-00003-of-00005.bin",
"model.layers.23.self_attn.rotary_emb.inv_freq": "pytorch_model-00003-of-00005.bin",
"model.layers.23.self_attn.v_proj.weight": "pytorch_model-00003-of-00005.bin",
"model.layers.24.input_layernorm.weight": "pytorch_model-00003-of-00005.bin",
"model.layers.24.mlp.down_proj.weight": "pytorch_model-00003-of-00005.bin",
"model.layers.24.mlp.gate_proj.weight": "pytorch_model-00003-of-00005.bin",
"model.layers.24.mlp.up_proj.weight": "pytorch_model-00003-of-00005.bin",
"model.layers.24.post_attention_layernorm.weight": "pytorch_model-00003-of-00005.bin",
"model.layers.24.self_attn.k_proj.weight": "pytorch_model-00003-of-00005.bin",
"model.layers.24.self_attn.o_proj.weight": "pytorch_model-00003-of-00005.bin",
"model.layers.24.self_attn.q_proj.weight": "pytorch_model-00003-of-00005.bin",
"model.layers.24.self_attn.rotary_emb.inv_freq": "pytorch_model-00003-of-00005.bin",
"model.layers.24.self_attn.v_proj.weight": "pytorch_model-00003-of-00005.bin",
"model.layers.25.input_layernorm.weight": "pytorch_model-00003-of-00005.bin",
"model.layers.25.mlp.down_proj.weight": "pytorch_model-00003-of-00005.bin",
"model.layers.25.mlp.gate_proj.weight": "pytorch_model-00003-of-00005.bin",
"model.layers.25.mlp.up_proj.weight": "pytorch_model-00003-of-00005.bin",
"model.layers.25.post_attention_layernorm.weight": "pytorch_model-00003-of-00005.bin",
"model.layers.25.self_attn.k_proj.weight": "pytorch_model-00003-of-00005.bin",
"model.layers.25.self_attn.o_proj.weight": "pytorch_model-00003-of-00005.bin",
"model.layers.25.self_attn.q_proj.weight": "pytorch_model-00003-of-00005.bin",
"model.layers.25.self_attn.rotary_emb.inv_freq": "pytorch_model-00003-of-00005.bin",
"model.layers.25.self_attn.v_proj.weight": "pytorch_model-00003-of-00005.bin",
"model.layers.26.input_layernorm.weight": "pytorch_model-00003-of-00005.bin",
"model.layers.26.mlp.down_proj.weight": "pytorch_model-00003-of-00005.bin",
"model.layers.26.mlp.gate_proj.weight": "pytorch_model-00003-of-00005.bin",
"model.layers.26.mlp.up_proj.weight": "pytorch_model-00003-of-00005.bin",
"model.layers.26.post_attention_layernorm.weight": "pytorch_model-00003-of-00005.bin",
"model.layers.26.self_attn.k_proj.weight": "pytorch_model-00003-of-00005.bin",
"model.layers.26.self_attn.o_proj.weight": "pytorch_model-00003-of-00005.bin",
"model.layers.26.self_attn.q_proj.weight": "pytorch_model-00003-of-00005.bin",
"model.layers.26.self_attn.rotary_emb.inv_freq": "pytorch_model-00003-of-00005.bin",
"model.layers.26.self_attn.v_proj.weight": "pytorch_model-00003-of-00005.bin",
"model.layers.27.input_layernorm.weight": "pytorch_model-00004-of-00005.bin",
"model.layers.27.mlp.down_proj.weight": "pytorch_model-00004-of-00005.bin",
"model.layers.27.mlp.gate_proj.weight": "pytorch_model-00004-of-00005.bin",
"model.layers.27.mlp.up_proj.weight": "pytorch_model-00004-of-00005.bin",
"model.layers.27.post_attention_layernorm.weight": "pytorch_model-00004-of-00005.bin",
"model.layers.27.self_attn.k_proj.weight": "pytorch_model-00003-of-00005.bin",
"model.layers.27.self_attn.o_proj.weight": "pytorch_model-00003-of-00005.bin",
"model.layers.27.self_attn.q_proj.weight": "pytorch_model-00003-of-00005.bin",
"model.layers.27.self_attn.rotary_emb.inv_freq": "pytorch_model-00003-of-00005.bin",
"model.layers.27.self_attn.v_proj.weight": "pytorch_model-00003-of-00005.bin",
"model.layers.28.input_layernorm.weight": "pytorch_model-00004-of-00005.bin",
"model.layers.28.mlp.down_proj.weight": "pytorch_model-00004-of-00005.bin",
"model.layers.28.mlp.gate_proj.weight": "pytorch_model-00004-of-00005.bin",
"model.layers.28.mlp.up_proj.weight": "pytorch_model-00004-of-00005.bin",
"model.layers.28.post_attention_layernorm.weight": "pytorch_model-00004-of-00005.bin",
"model.layers.28.self_attn.k_proj.weight": "pytorch_model-00004-of-00005.bin",
"model.layers.28.self_attn.o_proj.weight": "pytorch_model-00004-of-00005.bin",
"model.layers.28.self_attn.q_proj.weight": "pytorch_model-00004-of-00005.bin",
"model.layers.28.self_attn.rotary_emb.inv_freq": "pytorch_model-00004-of-00005.bin",
"model.layers.28.self_attn.v_proj.weight": "pytorch_model-00004-of-00005.bin",
"model.layers.29.input_layernorm.weight": "pytorch_model-00004-of-00005.bin",
"model.layers.29.mlp.down_proj.weight": "pytorch_model-00004-of-00005.bin",
"model.layers.29.mlp.gate_proj.weight": "pytorch_model-00004-of-00005.bin",
"model.layers.29.mlp.up_proj.weight": "pytorch_model-00004-of-00005.bin",
"model.layers.29.post_attention_layernorm.weight": "pytorch_model-00004-of-00005.bin",
"model.layers.29.self_attn.k_proj.weight": "pytorch_model-00004-of-00005.bin",
"model.layers.29.self_attn.o_proj.weight": "pytorch_model-00004-of-00005.bin",
"model.layers.29.self_attn.q_proj.weight": "pytorch_model-00004-of-00005.bin",
"model.layers.29.self_attn.rotary_emb.inv_freq": "pytorch_model-00004-of-00005.bin",
"model.layers.29.self_attn.v_proj.weight": "pytorch_model-00004-of-00005.bin",
"model.layers.3.input_layernorm.weight": "pytorch_model-00001-of-00005.bin",
"model.layers.3.mlp.down_proj.weight": "pytorch_model-00001-of-00005.bin",
"model.layers.3.mlp.gate_proj.weight": "pytorch_model-00001-of-00005.bin",
"model.layers.3.mlp.up_proj.weight": "pytorch_model-00001-of-00005.bin",
"model.layers.3.post_attention_layernorm.weight": "pytorch_model-00001-of-00005.bin",
"model.layers.3.self_attn.k_proj.weight": "pytorch_model-00001-of-00005.bin",
"model.layers.3.self_attn.o_proj.weight": "pytorch_model-00001-of-00005.bin",
"model.layers.3.self_attn.q_proj.weight": "pytorch_model-00001-of-00005.bin",
"model.layers.3.self_attn.rotary_emb.inv_freq": "pytorch_model-00001-of-00005.bin",
"model.layers.3.self_attn.v_proj.weight": "pytorch_model-00001-of-00005.bin",
"model.layers.30.input_layernorm.weight": "pytorch_model-00004-of-00005.bin",
"model.layers.30.mlp.down_proj.weight": "pytorch_model-00004-of-00005.bin",
"model.layers.30.mlp.gate_proj.weight": "pytorch_model-00004-of-00005.bin",
"model.layers.30.mlp.up_proj.weight": "pytorch_model-00004-of-00005.bin",
"model.layers.30.post_attention_layernorm.weight": "pytorch_model-00004-of-00005.bin",
"model.layers.30.self_attn.k_proj.weight": "pytorch_model-00004-of-00005.bin",
"model.layers.30.self_attn.o_proj.weight": "pytorch_model-00004-of-00005.bin",
"model.layers.30.self_attn.q_proj.weight": "pytorch_model-00004-of-00005.bin",
"model.layers.30.self_attn.rotary_emb.inv_freq": "pytorch_model-00004-of-00005.bin",
"model.layers.30.self_attn.v_proj.weight": "pytorch_model-00004-of-00005.bin",
"model.layers.31.input_layernorm.weight": "pytorch_model-00004-of-00005.bin",
"model.layers.31.mlp.down_proj.weight": "pytorch_model-00004-of-00005.bin",
"model.layers.31.mlp.gate_proj.weight": "pytorch_model-00004-of-00005.bin",
"model.layers.31.mlp.up_proj.weight": "pytorch_model-00004-of-00005.bin",
"model.layers.31.post_attention_layernorm.weight": "pytorch_model-00004-of-00005.bin",
"model.layers.31.self_attn.k_proj.weight": "pytorch_model-00004-of-00005.bin",
"model.layers.31.self_attn.o_proj.weight": "pytorch_model-00004-of-00005.bin",
"model.layers.31.self_attn.q_proj.weight": "pytorch_model-00004-of-00005.bin",
"model.layers.31.self_attn.rotary_emb.inv_freq": "pytorch_model-00004-of-00005.bin",
"model.layers.31.self_attn.v_proj.weight": "pytorch_model-00004-of-00005.bin",
"model.layers.32.input_layernorm.weight": "pytorch_model-00004-of-00005.bin",
"model.layers.32.mlp.down_proj.weight": "pytorch_model-00004-of-00005.bin",
"model.layers.32.mlp.gate_proj.weight": "pytorch_model-00004-of-00005.bin",
"model.layers.32.mlp.up_proj.weight": "pytorch_model-00004-of-00005.bin",
"model.layers.32.post_attention_layernorm.weight": "pytorch_model-00004-of-00005.bin",
"model.layers.32.self_attn.k_proj.weight": "pytorch_model-00004-of-00005.bin",
"model.layers.32.self_attn.o_proj.weight": "pytorch_model-00004-of-00005.bin",
"model.layers.32.self_attn.q_proj.weight": "pytorch_model-00004-of-00005.bin",
"model.layers.32.self_attn.rotary_emb.inv_freq": "pytorch_model-00004-of-00005.bin",
"model.layers.32.self_attn.v_proj.weight": "pytorch_model-00004-of-00005.bin",
"model.layers.33.input_layernorm.weight": "pytorch_model-00004-of-00005.bin",
"model.layers.33.mlp.down_proj.weight": "pytorch_model-00004-of-00005.bin",
"model.layers.33.mlp.gate_proj.weight": "pytorch_model-00004-of-00005.bin",
"model.layers.33.mlp.up_proj.weight": "pytorch_model-00004-of-00005.bin",
"model.layers.33.post_attention_layernorm.weight": "pytorch_model-00004-of-00005.bin",
"model.layers.33.self_attn.k_proj.weight": "pytorch_model-00004-of-00005.bin",
"model.layers.33.self_attn.o_proj.weight": "pytorch_model-00004-of-00005.bin",
"model.layers.33.self_attn.q_proj.weight": "pytorch_model-00004-of-00005.bin",
"model.layers.33.self_attn.rotary_emb.inv_freq": "pytorch_model-00004-of-00005.bin",
"model.layers.33.self_attn.v_proj.weight": "pytorch_model-00004-of-00005.bin",
"model.layers.34.input_layernorm.weight": "pytorch_model-00004-of-00005.bin",
"model.layers.34.mlp.down_proj.weight": "pytorch_model-00004-of-00005.bin",
"model.layers.34.mlp.gate_proj.weight": "pytorch_model-00004-of-00005.bin",
"model.layers.34.mlp.up_proj.weight": "pytorch_model-00004-of-00005.bin",
"model.layers.34.post_attention_layernorm.weight": "pytorch_model-00004-of-00005.bin",
"model.layers.34.self_attn.k_proj.weight": "pytorch_model-00004-of-00005.bin",
"model.layers.34.self_attn.o_proj.weight": "pytorch_model-00004-of-00005.bin",
"model.layers.34.self_attn.q_proj.weight": "pytorch_model-00004-of-00005.bin",
"model.layers.34.self_attn.rotary_emb.inv_freq": "pytorch_model-00004-of-00005.bin",
"model.layers.34.self_attn.v_proj.weight": "pytorch_model-00004-of-00005.bin",
"model.layers.35.input_layernorm.weight": "pytorch_model-00004-of-00005.bin",
"model.layers.35.mlp.down_proj.weight": "pytorch_model-00004-of-00005.bin",
"model.layers.35.mlp.gate_proj.weight": "pytorch_model-00004-of-00005.bin",
"model.layers.35.mlp.up_proj.weight": "pytorch_model-00004-of-00005.bin",
"model.layers.35.post_attention_layernorm.weight": "pytorch_model-00004-of-00005.bin",
"model.layers.35.self_attn.k_proj.weight": "pytorch_model-00004-of-00005.bin",
"model.layers.35.self_attn.o_proj.weight": "pytorch_model-00004-of-00005.bin",
"model.layers.35.self_attn.q_proj.weight": "pytorch_model-00004-of-00005.bin",
"model.layers.35.self_attn.rotary_emb.inv_freq": "pytorch_model-00004-of-00005.bin",
"model.layers.35.self_attn.v_proj.weight": "pytorch_model-00004-of-00005.bin",
"model.layers.36.input_layernorm.weight": "pytorch_model-00005-of-00005.bin",
"model.layers.36.mlp.down_proj.weight": "pytorch_model-00005-of-00005.bin",
"model.layers.36.mlp.gate_proj.weight": "pytorch_model-00004-of-00005.bin",
"model.layers.36.mlp.up_proj.weight": "pytorch_model-00005-of-00005.bin",
"model.layers.36.post_attention_layernorm.weight": "pytorch_model-00005-of-00005.bin",
"model.layers.36.self_attn.k_proj.weight": "pytorch_model-00004-of-00005.bin",
"model.layers.36.self_attn.o_proj.weight": "pytorch_model-00004-of-00005.bin",
"model.layers.36.self_attn.q_proj.weight": "pytorch_model-00004-of-00005.bin",
"model.layers.36.self_attn.rotary_emb.inv_freq": "pytorch_model-00004-of-00005.bin",
"model.layers.36.self_attn.v_proj.weight": "pytorch_model-00004-of-00005.bin",
"model.layers.37.input_layernorm.weight": "pytorch_model-00005-of-00005.bin",
"model.layers.37.mlp.down_proj.weight": "pytorch_model-00005-of-00005.bin",
"model.layers.37.mlp.gate_proj.weight": "pytorch_model-00005-of-00005.bin",
"model.layers.37.mlp.up_proj.weight": "pytorch_model-00005-of-00005.bin",
"model.layers.37.post_attention_layernorm.weight": "pytorch_model-00005-of-00005.bin",
"model.layers.37.self_attn.k_proj.weight": "pytorch_model-00005-of-00005.bin",
"model.layers.37.self_attn.o_proj.weight": "pytorch_model-00005-of-00005.bin",
"model.layers.37.self_attn.q_proj.weight": "pytorch_model-00005-of-00005.bin",
"model.layers.37.self_attn.rotary_emb.inv_freq": "pytorch_model-00005-of-00005.bin",
"model.layers.37.self_attn.v_proj.weight": "pytorch_model-00005-of-00005.bin",
"model.layers.38.input_layernorm.weight": "pytorch_model-00005-of-00005.bin",
"model.layers.38.mlp.down_proj.weight": "pytorch_model-00005-of-00005.bin",
"model.layers.38.mlp.gate_proj.weight": "pytorch_model-00005-of-00005.bin",
"model.layers.38.mlp.up_proj.weight": "pytorch_model-00005-of-00005.bin",
"model.layers.38.post_attention_layernorm.weight": "pytorch_model-00005-of-00005.bin",
"model.layers.38.self_attn.k_proj.weight": "pytorch_model-00005-of-00005.bin",
"model.layers.38.self_attn.o_proj.weight": "pytorch_model-00005-of-00005.bin",
"model.layers.38.self_attn.q_proj.weight": "pytorch_model-00005-of-00005.bin",
"model.layers.38.self_attn.rotary_emb.inv_freq": "pytorch_model-00005-of-00005.bin",
"model.layers.38.self_attn.v_proj.weight": "pytorch_model-00005-of-00005.bin",
"model.layers.39.input_layernorm.weight": "pytorch_model-00005-of-00005.bin",
"model.layers.39.mlp.down_proj.weight": "pytorch_model-00005-of-00005.bin",
"model.layers.39.mlp.gate_proj.weight": "pytorch_model-00005-of-00005.bin",
"model.layers.39.mlp.up_proj.weight": "pytorch_model-00005-of-00005.bin",
"model.layers.39.post_attention_layernorm.weight": "pytorch_model-00005-of-00005.bin",
"model.layers.39.self_attn.k_proj.weight": "pytorch_model-00005-of-00005.bin",
"model.layers.39.self_attn.o_proj.weight": "pytorch_model-00005-of-00005.bin",
"model.layers.39.self_attn.q_proj.weight": "pytorch_model-00005-of-00005.bin",
"model.layers.39.self_attn.rotary_emb.inv_freq": "pytorch_model-00005-of-00005.bin",
"model.layers.39.self_attn.v_proj.weight": "pytorch_model-00005-of-00005.bin",
"model.layers.4.input_layernorm.weight": "pytorch_model-00001-of-00005.bin",
"model.layers.4.mlp.down_proj.weight": "pytorch_model-00001-of-00005.bin",
"model.layers.4.mlp.gate_proj.weight": "pytorch_model-00001-of-00005.bin",
"model.layers.4.mlp.up_proj.weight": "pytorch_model-00001-of-00005.bin",
"model.layers.4.post_attention_layernorm.weight": "pytorch_model-00001-of-00005.bin",
"model.layers.4.self_attn.k_proj.weight": "pytorch_model-00001-of-00005.bin",
"model.layers.4.self_attn.o_proj.weight": "pytorch_model-00001-of-00005.bin",
"model.layers.4.self_attn.q_proj.weight": "pytorch_model-00001-of-00005.bin",
"model.layers.4.self_attn.rotary_emb.inv_freq": "pytorch_model-00001-of-00005.bin",
"model.layers.4.self_attn.v_proj.weight": "pytorch_model-00001-of-00005.bin",
"model.layers.5.input_layernorm.weight": "pytorch_model-00001-of-00005.bin",
"model.layers.5.mlp.down_proj.weight": "pytorch_model-00001-of-00005.bin",
"model.layers.5.mlp.gate_proj.weight": "pytorch_model-00001-of-00005.bin",
"model.layers.5.mlp.up_proj.weight": "pytorch_model-00001-of-00005.bin",
"model.layers.5.post_attention_layernorm.weight": "pytorch_model-00001-of-00005.bin",
"model.layers.5.self_attn.k_proj.weight": "pytorch_model-00001-of-00005.bin",
"model.layers.5.self_attn.o_proj.weight": "pytorch_model-00001-of-00005.bin",
"model.layers.5.self_attn.q_proj.weight": "pytorch_model-00001-of-00005.bin",
"model.layers.5.self_attn.rotary_emb.inv_freq": "pytorch_model-00001-of-00005.bin",
"model.layers.5.self_attn.v_proj.weight": "pytorch_model-00001-of-00005.bin",
"model.layers.6.input_layernorm.weight": "pytorch_model-00001-of-00005.bin",
"model.layers.6.mlp.down_proj.weight": "pytorch_model-00001-of-00005.bin",
"model.layers.6.mlp.gate_proj.weight": "pytorch_model-00001-of-00005.bin",
"model.layers.6.mlp.up_proj.weight": "pytorch_model-00001-of-00005.bin",
"model.layers.6.post_attention_layernorm.weight": "pytorch_model-00001-of-00005.bin",
"model.layers.6.self_attn.k_proj.weight": "pytorch_model-00001-of-00005.bin",
"model.layers.6.self_attn.o_proj.weight": "pytorch_model-00001-of-00005.bin",
"model.layers.6.self_attn.q_proj.weight": "pytorch_model-00001-of-00005.bin",
"model.layers.6.self_attn.rotary_emb.inv_freq": "pytorch_model-00001-of-00005.bin",
"model.layers.6.self_attn.v_proj.weight": "pytorch_model-00001-of-00005.bin",
"model.layers.7.input_layernorm.weight": "pytorch_model-00001-of-00005.bin",
"model.layers.7.mlp.down_proj.weight": "pytorch_model-00001-of-00005.bin",
"model.layers.7.mlp.gate_proj.weight": "pytorch_model-00001-of-00005.bin",
"model.layers.7.mlp.up_proj.weight": "pytorch_model-00001-of-00005.bin",
"model.layers.7.post_attention_layernorm.weight": "pytorch_model-00001-of-00005.bin",
"model.layers.7.self_attn.k_proj.weight": "pytorch_model-00001-of-00005.bin",
"model.layers.7.self_attn.o_proj.weight": "pytorch_model-00001-of-00005.bin",
"model.layers.7.self_attn.q_proj.weight": "pytorch_model-00001-of-00005.bin",
"model.layers.7.self_attn.rotary_emb.inv_freq": "pytorch_model-00001-of-00005.bin",
"model.layers.7.self_attn.v_proj.weight": "pytorch_model-00001-of-00005.bin",
"model.layers.8.input_layernorm.weight": "pytorch_model-00002-of-00005.bin",
"model.layers.8.mlp.down_proj.weight": "pytorch_model-00002-of-00005.bin",
"model.layers.8.mlp.gate_proj.weight": "pytorch_model-00001-of-00005.bin",
"model.layers.8.mlp.up_proj.weight": "pytorch_model-00001-of-00005.bin",
"model.layers.8.post_attention_layernorm.weight": "pytorch_model-00002-of-00005.bin",
"model.layers.8.self_attn.k_proj.weight": "pytorch_model-00001-of-00005.bin",
"model.layers.8.self_attn.o_proj.weight": "pytorch_model-00001-of-00005.bin",
"model.layers.8.self_attn.q_proj.weight": "pytorch_model-00001-of-00005.bin",
"model.layers.8.self_attn.rotary_emb.inv_freq": "pytorch_model-00001-of-00005.bin",
"model.layers.8.self_attn.v_proj.weight": "pytorch_model-00001-of-00005.bin",
"model.layers.9.input_layernorm.weight": "pytorch_model-00002-of-00005.bin",
"model.layers.9.mlp.down_proj.weight": "pytorch_model-00002-of-00005.bin",
"model.layers.9.mlp.gate_proj.weight": "pytorch_model-00002-of-00005.bin",
"model.layers.9.mlp.up_proj.weight": "pytorch_model-00002-of-00005.bin",
"model.layers.9.post_attention_layernorm.weight": "pytorch_model-00002-of-00005.bin",
"model.layers.9.self_attn.k_proj.weight": "pytorch_model-00002-of-00005.bin",
"model.layers.9.self_attn.o_proj.weight": "pytorch_model-00002-of-00005.bin",
"model.layers.9.self_attn.q_proj.weight": "pytorch_model-00002-of-00005.bin",
"model.layers.9.self_attn.rotary_emb.inv_freq": "pytorch_model-00002-of-00005.bin",
"model.layers.9.self_attn.v_proj.weight": "pytorch_model-00002-of-00005.bin",
"model.norm.weight": "pytorch_model-00005-of-00005.bin"
}
}

24
special_tokens_map.json Normal file
View File

@@ -0,0 +1,24 @@
{
"bos_token": {
"content": "<s>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false
},
"eos_token": {
"content": "</s>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false
},
"pad_token": "<unk>",
"unk_token": {
"content": "<unk>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false
}
}

BIN
tokenizer.model (Stored with Git LFS) Normal file
View File

Binary file not shown.

34
tokenizer_config.json Normal file
View File

@@ -0,0 +1,34 @@
{
"add_bos_token": true,
"add_eos_token": false,
"bos_token": {
"__type": "AddedToken",
"content": "<s>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false
},
"clean_up_tokenization_spaces": false,
"eos_token": {
"__type": "AddedToken",
"content": "</s>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false
},
"legacy": false,
"model_max_length": 1000000000000000019884624838656,
"pad_token": null,
"sp_model_kwargs": {},
"tokenizer_class": "LlamaTokenizer",
"unk_token": {
"__type": "AddedToken",
"content": "<unk>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false
}
}