naxautify/pythia-1.4b-deduped-8k
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

初始化项目，由ModelHub XC社区提供模型

Browse Source 
Model: naxautify/pythia-1.4b-deduped-8k
Source: Original Platform
This commit is contained in:
ModelHub XC
2026-07-23 18:36:12 +08:00
commit ba39edf434
21 changed files with 101396 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
.gitattributes vendored Normal file
View File

@@ -0,0 +1,34 @@
*.7z filter=lfs diff=lfs merge=lfs -text
*.arrow filter=lfs diff=lfs merge=lfs -text
*.bin filter=lfs diff=lfs merge=lfs -text
*.bz2 filter=lfs diff=lfs merge=lfs -text
*.ckpt filter=lfs diff=lfs merge=lfs -text
*.ftz filter=lfs diff=lfs merge=lfs -text
*.gz filter=lfs diff=lfs merge=lfs -text
*.h5 filter=lfs diff=lfs merge=lfs -text
*.joblib filter=lfs diff=lfs merge=lfs -text
*.lfs.* filter=lfs diff=lfs merge=lfs -text
*.mlmodel filter=lfs diff=lfs merge=lfs -text
*.model filter=lfs diff=lfs merge=lfs -text
*.msgpack filter=lfs diff=lfs merge=lfs -text
*.npy filter=lfs diff=lfs merge=lfs -text
*.npz filter=lfs diff=lfs merge=lfs -text
*.onnx filter=lfs diff=lfs merge=lfs -text
*.ot filter=lfs diff=lfs merge=lfs -text
*.parquet filter=lfs diff=lfs merge=lfs -text
*.pb filter=lfs diff=lfs merge=lfs -text
*.pickle filter=lfs diff=lfs merge=lfs -text
*.pkl filter=lfs diff=lfs merge=lfs -text
*.pt filter=lfs diff=lfs merge=lfs -text
*.pth filter=lfs diff=lfs merge=lfs -text
*.rar filter=lfs diff=lfs merge=lfs -text
*.safetensors filter=lfs diff=lfs merge=lfs -text
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
*.tar.* filter=lfs diff=lfs merge=lfs -text
*.tflite filter=lfs diff=lfs merge=lfs -text
*.tgz filter=lfs diff=lfs merge=lfs -text
*.wasm filter=lfs diff=lfs merge=lfs -text
*.xz filter=lfs diff=lfs merge=lfs -text
*.zip filter=lfs diff=lfs merge=lfs -text
*.zst filter=lfs diff=lfs merge=lfs -text
*tfevents* filter=lfs diff=lfs merge=lfs -text

10
README.md Normal file
View File

@@ -0,0 +1,10 @@
---
datasets:
- EleutherAI/the_pile_deduplicated
pipeline_tag: text-generation
library_name: transformers
---
# Pythia 1.4b Deduped with 8k Context Window
This model fine-tunes Pythia 1.4b model with a context window of 8k tokens. With optimizations like Flash Attention & bitsandbytes, I could fit the model the entire model with a batch size of 1, on a single A100 (40 GB). The fine-tuning took ~30 hours, after which the loss was similar to that of fine-tuning at the context window of 2k tokens.

25
config.json Normal file
View File

@@ -0,0 +1,25 @@
{
"_name_or_path": "EleutherAI/pythia-1.4b-deduped",
"architectures": [
"GPTNeoXForCausalLM"
],
"bos_token_id": 0,
"eos_token_id": 0,
"hidden_act": "gelu",
"hidden_size": 2048,
"initializer_range": 0.02,
"intermediate_size": 8192,
"layer_norm_eps": 1e-05,
"max_position_embeddings": 2048,
"model_type": "gpt_neox",
"num_attention_heads": 16,
"num_hidden_layers": 24,
"rotary_emb_base": 10000,
"rotary_pct": 0.25,
"tie_word_embeddings": false,
"torch_dtype": "float16",
"transformers_version": "4.28.1",
"use_cache": true,
"use_parallel_residual": true,
"vocab_size": 50304
}

6
generation_config.json Normal file
View File

@@ -0,0 +1,6 @@
{
"_from_model_config": true,
"bos_token_id": 0,
"eos_token_id": 0,
"transformers_version": "4.28.1"
}

3
model-00001-of-00006.safetensors Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:46f38ecb18644e74132db9480edf2b33bba7a6ef6789b6848b1bda14f0651455
size 487224374

3
model-00002-of-00006.safetensors Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:cd3aacd9401c85662e17aab2f2afeec2317ce6e62296042544f0387b89427d46
size 490991978

3
model-00003-of-00006.safetensors Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:babf275e9f3d25aff596e27bdaa083379fb1884e9b012d9e33344c9aac5f9e6e
size 516170122

3
model-00004-of-00006.safetensors Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:e6fc686e8a149fbfd3d6f68688b1fb8b57781fddc31d21dac4edd364d084282b
size 499384738

3
model-00005-of-00006.safetensors Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:723494bd010386731b614ffc18e1d937ecc45bd33f17a63f4b988976ca5d0bbc
size 486792816

3
model-00006-of-00006.safetensors Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:271cf173416001ca9ac0e782373d4157a2316105fbb9d37a3ccb7366aca76c2f
size 449437700

371
model.safetensors.index.json Normal file
View File

@@ -0,0 +1,371 @@
{
"metadata": {
"total_size": 2841879344.0
},
"weight_map": {
"embed_out.weight": "model-00006-of-00006.safetensors",
"gpt_neox.embed_in.weight": "model-00001-of-00006.safetensors",
"gpt_neox.final_layer_norm.bias": "model-00006-of-00006.safetensors",
"gpt_neox.final_layer_norm.weight": "model-00006-of-00006.safetensors",
"gpt_neox.layers.0.attention.bias": "model-00001-of-00006.safetensors",
"gpt_neox.layers.0.attention.dense.bias": "model-00001-of-00006.safetensors",
"gpt_neox.layers.0.attention.dense.weight": "model-00001-of-00006.safetensors",
"gpt_neox.layers.0.attention.masked_bias": "model-00001-of-00006.safetensors",
"gpt_neox.layers.0.attention.query_key_value.bias": "model-00001-of-00006.safetensors",
"gpt_neox.layers.0.attention.query_key_value.weight": "model-00001-of-00006.safetensors",
"gpt_neox.layers.0.attention.rotary_emb.inv_freq": "model-00001-of-00006.safetensors",
"gpt_neox.layers.0.input_layernorm.bias": "model-00001-of-00006.safetensors",
"gpt_neox.layers.0.input_layernorm.weight": "model-00001-of-00006.safetensors",
"gpt_neox.layers.0.mlp.dense_4h_to_h.bias": "model-00001-of-00006.safetensors",
"gpt_neox.layers.0.mlp.dense_4h_to_h.weight": "model-00001-of-00006.safetensors",
"gpt_neox.layers.0.mlp.dense_h_to_4h.bias": "model-00001-of-00006.safetensors",
"gpt_neox.layers.0.mlp.dense_h_to_4h.weight": "model-00001-of-00006.safetensors",
"gpt_neox.layers.0.post_attention_layernorm.bias": "model-00001-of-00006.safetensors",
"gpt_neox.layers.0.post_attention_layernorm.weight": "model-00001-of-00006.safetensors",
"gpt_neox.layers.1.attention.bias": "model-00001-of-00006.safetensors",
"gpt_neox.layers.1.attention.dense.bias": "model-00001-of-00006.safetensors",
"gpt_neox.layers.1.attention.dense.weight": "model-00001-of-00006.safetensors",
"gpt_neox.layers.1.attention.masked_bias": "model-00001-of-00006.safetensors",
"gpt_neox.layers.1.attention.query_key_value.bias": "model-00001-of-00006.safetensors",
"gpt_neox.layers.1.attention.query_key_value.weight": "model-00001-of-00006.safetensors",
"gpt_neox.layers.1.attention.rotary_emb.inv_freq": "model-00001-of-00006.safetensors",
"gpt_neox.layers.1.input_layernorm.bias": "model-00001-of-00006.safetensors",
"gpt_neox.layers.1.input_layernorm.weight": "model-00001-of-00006.safetensors",
"gpt_neox.layers.1.mlp.dense_4h_to_h.bias": "model-00001-of-00006.safetensors",
"gpt_neox.layers.1.mlp.dense_4h_to_h.weight": "model-00001-of-00006.safetensors",
"gpt_neox.layers.1.mlp.dense_h_to_4h.bias": "model-00001-of-00006.safetensors",
"gpt_neox.layers.1.mlp.dense_h_to_4h.weight": "model-00001-of-00006.safetensors",
"gpt_neox.layers.1.post_attention_layernorm.bias": "model-00001-of-00006.safetensors",
"gpt_neox.layers.1.post_attention_layernorm.weight": "model-00001-of-00006.safetensors",
"gpt_neox.layers.10.attention.bias": "model-00003-of-00006.safetensors",
"gpt_neox.layers.10.attention.dense.bias": "model-00003-of-00006.safetensors",
"gpt_neox.layers.10.attention.dense.weight": "model-00003-of-00006.safetensors",
"gpt_neox.layers.10.attention.masked_bias": "model-00003-of-00006.safetensors",
"gpt_neox.layers.10.attention.query_key_value.bias": "model-00003-of-00006.safetensors",
"gpt_neox.layers.10.attention.query_key_value.weight": "model-00003-of-00006.safetensors",
"gpt_neox.layers.10.attention.rotary_emb.inv_freq": "model-00003-of-00006.safetensors",
"gpt_neox.layers.10.input_layernorm.bias": "model-00003-of-00006.safetensors",
"gpt_neox.layers.10.input_layernorm.weight": "model-00003-of-00006.safetensors",
"gpt_neox.layers.10.mlp.dense_4h_to_h.bias": "model-00003-of-00006.safetensors",
"gpt_neox.layers.10.mlp.dense_4h_to_h.weight": "model-00003-of-00006.safetensors",
"gpt_neox.layers.10.mlp.dense_h_to_4h.bias": "model-00003-of-00006.safetensors",
"gpt_neox.layers.10.mlp.dense_h_to_4h.weight": "model-00003-of-00006.safetensors",
"gpt_neox.layers.10.post_attention_layernorm.bias": "model-00003-of-00006.safetensors",
"gpt_neox.layers.10.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
"gpt_neox.layers.11.attention.bias": "model-00003-of-00006.safetensors",
"gpt_neox.layers.11.attention.dense.bias": "model-00003-of-00006.safetensors",
"gpt_neox.layers.11.attention.dense.weight": "model-00003-of-00006.safetensors",
"gpt_neox.layers.11.attention.masked_bias": "model-00003-of-00006.safetensors",
"gpt_neox.layers.11.attention.query_key_value.bias": "model-00003-of-00006.safetensors",
"gpt_neox.layers.11.attention.query_key_value.weight": "model-00003-of-00006.safetensors",
"gpt_neox.layers.11.attention.rotary_emb.inv_freq": "model-00003-of-00006.safetensors",
"gpt_neox.layers.11.input_layernorm.bias": "model-00003-of-00006.safetensors",
"gpt_neox.layers.11.input_layernorm.weight": "model-00003-of-00006.safetensors",
"gpt_neox.layers.11.mlp.dense_4h_to_h.bias": "model-00003-of-00006.safetensors",
"gpt_neox.layers.11.mlp.dense_4h_to_h.weight": "model-00003-of-00006.safetensors",
"gpt_neox.layers.11.mlp.dense_h_to_4h.bias": "model-00003-of-00006.safetensors",
"gpt_neox.layers.11.mlp.dense_h_to_4h.weight": "model-00003-of-00006.safetensors",
"gpt_neox.layers.11.post_attention_layernorm.bias": "model-00003-of-00006.safetensors",
"gpt_neox.layers.11.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
"gpt_neox.layers.12.attention.bias": "model-00003-of-00006.safetensors",
"gpt_neox.layers.12.attention.dense.bias": "model-00004-of-00006.safetensors",
"gpt_neox.layers.12.attention.dense.weight": "model-00004-of-00006.safetensors",
"gpt_neox.layers.12.attention.masked_bias": "model-00003-of-00006.safetensors",
"gpt_neox.layers.12.attention.query_key_value.bias": "model-00003-of-00006.safetensors",
"gpt_neox.layers.12.attention.query_key_value.weight": "model-00003-of-00006.safetensors",
"gpt_neox.layers.12.attention.rotary_emb.inv_freq": "model-00003-of-00006.safetensors",
"gpt_neox.layers.12.input_layernorm.bias": "model-00003-of-00006.safetensors",
"gpt_neox.layers.12.input_layernorm.weight": "model-00003-of-00006.safetensors",
"gpt_neox.layers.12.mlp.dense_4h_to_h.bias": "model-00004-of-00006.safetensors",
"gpt_neox.layers.12.mlp.dense_4h_to_h.weight": "model-00004-of-00006.safetensors",
"gpt_neox.layers.12.mlp.dense_h_to_4h.bias": "model-00004-of-00006.safetensors",
"gpt_neox.layers.12.mlp.dense_h_to_4h.weight": "model-00004-of-00006.safetensors",
"gpt_neox.layers.12.post_attention_layernorm.bias": "model-00003-of-00006.safetensors",
"gpt_neox.layers.12.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
"gpt_neox.layers.13.attention.bias": "model-00004-of-00006.safetensors",
"gpt_neox.layers.13.attention.dense.bias": "model-00004-of-00006.safetensors",
"gpt_neox.layers.13.attention.dense.weight": "model-00004-of-00006.safetensors",
"gpt_neox.layers.13.attention.masked_bias": "model-00004-of-00006.safetensors",
"gpt_neox.layers.13.attention.query_key_value.bias": "model-00004-of-00006.safetensors",
"gpt_neox.layers.13.attention.query_key_value.weight": "model-00004-of-00006.safetensors",
"gpt_neox.layers.13.attention.rotary_emb.inv_freq": "model-00004-of-00006.safetensors",
"gpt_neox.layers.13.input_layernorm.bias": "model-00004-of-00006.safetensors",
"gpt_neox.layers.13.input_layernorm.weight": "model-00004-of-00006.safetensors",
"gpt_neox.layers.13.mlp.dense_4h_to_h.bias": "model-00004-of-00006.safetensors",
"gpt_neox.layers.13.mlp.dense_4h_to_h.weight": "model-00004-of-00006.safetensors",
"gpt_neox.layers.13.mlp.dense_h_to_4h.bias": "model-00004-of-00006.safetensors",
"gpt_neox.layers.13.mlp.dense_h_to_4h.weight": "model-00004-of-00006.safetensors",
"gpt_neox.layers.13.post_attention_layernorm.bias": "model-00004-of-00006.safetensors",
"gpt_neox.layers.13.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
"gpt_neox.layers.14.attention.bias": "model-00004-of-00006.safetensors",
"gpt_neox.layers.14.attention.dense.bias": "model-00004-of-00006.safetensors",
"gpt_neox.layers.14.attention.dense.weight": "model-00004-of-00006.safetensors",
"gpt_neox.layers.14.attention.masked_bias": "model-00004-of-00006.safetensors",
"gpt_neox.layers.14.attention.query_key_value.bias": "model-00004-of-00006.safetensors",
"gpt_neox.layers.14.attention.query_key_value.weight": "model-00004-of-00006.safetensors",
"gpt_neox.layers.14.attention.rotary_emb.inv_freq": "model-00004-of-00006.safetensors",
"gpt_neox.layers.14.input_layernorm.bias": "model-00004-of-00006.safetensors",
"gpt_neox.layers.14.input_layernorm.weight": "model-00004-of-00006.safetensors",
"gpt_neox.layers.14.mlp.dense_4h_to_h.bias": "model-00004-of-00006.safetensors",
"gpt_neox.layers.14.mlp.dense_4h_to_h.weight": "model-00004-of-00006.safetensors",
"gpt_neox.layers.14.mlp.dense_h_to_4h.bias": "model-00004-of-00006.safetensors",
"gpt_neox.layers.14.mlp.dense_h_to_4h.weight": "model-00004-of-00006.safetensors",
"gpt_neox.layers.14.post_attention_layernorm.bias": "model-00004-of-00006.safetensors",
"gpt_neox.layers.14.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
"gpt_neox.layers.15.attention.bias": "model-00004-of-00006.safetensors",
"gpt_neox.layers.15.attention.dense.bias": "model-00004-of-00006.safetensors",
"gpt_neox.layers.15.attention.dense.weight": "model-00004-of-00006.safetensors",
"gpt_neox.layers.15.attention.masked_bias": "model-00004-of-00006.safetensors",
"gpt_neox.layers.15.attention.query_key_value.bias": "model-00004-of-00006.safetensors",
"gpt_neox.layers.15.attention.query_key_value.weight": "model-00004-of-00006.safetensors",
"gpt_neox.layers.15.attention.rotary_emb.inv_freq": "model-00004-of-00006.safetensors",
"gpt_neox.layers.15.input_layernorm.bias": "model-00004-of-00006.safetensors",
"gpt_neox.layers.15.input_layernorm.weight": "model-00004-of-00006.safetensors",
"gpt_neox.layers.15.mlp.dense_4h_to_h.bias": "model-00004-of-00006.safetensors",
"gpt_neox.layers.15.mlp.dense_4h_to_h.weight": "model-00004-of-00006.safetensors",
"gpt_neox.layers.15.mlp.dense_h_to_4h.bias": "model-00004-of-00006.safetensors",
"gpt_neox.layers.15.mlp.dense_h_to_4h.weight": "model-00004-of-00006.safetensors",
"gpt_neox.layers.15.post_attention_layernorm.bias": "model-00004-of-00006.safetensors",
"gpt_neox.layers.15.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
"gpt_neox.layers.16.attention.bias": "model-00004-of-00006.safetensors",
"gpt_neox.layers.16.attention.dense.bias": "model-00004-of-00006.safetensors",
"gpt_neox.layers.16.attention.dense.weight": "model-00004-of-00006.safetensors",
"gpt_neox.layers.16.attention.masked_bias": "model-00004-of-00006.safetensors",
"gpt_neox.layers.16.attention.query_key_value.bias": "model-00004-of-00006.safetensors",
"gpt_neox.layers.16.attention.query_key_value.weight": "model-00004-of-00006.safetensors",
"gpt_neox.layers.16.attention.rotary_emb.inv_freq": "model-00004-of-00006.safetensors",
"gpt_neox.layers.16.input_layernorm.bias": "model-00004-of-00006.safetensors",
"gpt_neox.layers.16.input_layernorm.weight": "model-00004-of-00006.safetensors",
"gpt_neox.layers.16.mlp.dense_4h_to_h.bias": "model-00004-of-00006.safetensors",
"gpt_neox.layers.16.mlp.dense_4h_to_h.weight": "model-00004-of-00006.safetensors",
"gpt_neox.layers.16.mlp.dense_h_to_4h.bias": "model-00004-of-00006.safetensors",
"gpt_neox.layers.16.mlp.dense_h_to_4h.weight": "model-00004-of-00006.safetensors",
"gpt_neox.layers.16.post_attention_layernorm.bias": "model-00004-of-00006.safetensors",
"gpt_neox.layers.16.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
"gpt_neox.layers.17.attention.bias": "model-00004-of-00006.safetensors",
"gpt_neox.layers.17.attention.dense.bias": "model-00005-of-00006.safetensors",
"gpt_neox.layers.17.attention.dense.weight": "model-00005-of-00006.safetensors",
"gpt_neox.layers.17.attention.masked_bias": "model-00004-of-00006.safetensors",
"gpt_neox.layers.17.attention.query_key_value.bias": "model-00005-of-00006.safetensors",
"gpt_neox.layers.17.attention.query_key_value.weight": "model-00005-of-00006.safetensors",
"gpt_neox.layers.17.attention.rotary_emb.inv_freq": "model-00004-of-00006.safetensors",
"gpt_neox.layers.17.input_layernorm.bias": "model-00004-of-00006.safetensors",
"gpt_neox.layers.17.input_layernorm.weight": "model-00004-of-00006.safetensors",
"gpt_neox.layers.17.mlp.dense_4h_to_h.bias": "model-00005-of-00006.safetensors",
"gpt_neox.layers.17.mlp.dense_4h_to_h.weight": "model-00005-of-00006.safetensors",
"gpt_neox.layers.17.mlp.dense_h_to_4h.bias": "model-00005-of-00006.safetensors",
"gpt_neox.layers.17.mlp.dense_h_to_4h.weight": "model-00005-of-00006.safetensors",
"gpt_neox.layers.17.post_attention_layernorm.bias": "model-00004-of-00006.safetensors",
"gpt_neox.layers.17.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
"gpt_neox.layers.18.attention.bias": "model-00005-of-00006.safetensors",
"gpt_neox.layers.18.attention.dense.bias": "model-00005-of-00006.safetensors",
"gpt_neox.layers.18.attention.dense.weight": "model-00005-of-00006.safetensors",
"gpt_neox.layers.18.attention.masked_bias": "model-00005-of-00006.safetensors",
"gpt_neox.layers.18.attention.query_key_value.bias": "model-00005-of-00006.safetensors",
"gpt_neox.layers.18.attention.query_key_value.weight": "model-00005-of-00006.safetensors",
"gpt_neox.layers.18.attention.rotary_emb.inv_freq": "model-00005-of-00006.safetensors",
"gpt_neox.layers.18.input_layernorm.bias": "model-00005-of-00006.safetensors",
"gpt_neox.layers.18.input_layernorm.weight": "model-00005-of-00006.safetensors",
"gpt_neox.layers.18.mlp.dense_4h_to_h.bias": "model-00005-of-00006.safetensors",
"gpt_neox.layers.18.mlp.dense_4h_to_h.weight": "model-00005-of-00006.safetensors",
"gpt_neox.layers.18.mlp.dense_h_to_4h.bias": "model-00005-of-00006.safetensors",
"gpt_neox.layers.18.mlp.dense_h_to_4h.weight": "model-00005-of-00006.safetensors",
"gpt_neox.layers.18.post_attention_layernorm.bias": "model-00005-of-00006.safetensors",
"gpt_neox.layers.18.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
"gpt_neox.layers.19.attention.bias": "model-00005-of-00006.safetensors",
"gpt_neox.layers.19.attention.dense.bias": "model-00005-of-00006.safetensors",
"gpt_neox.layers.19.attention.dense.weight": "model-00005-of-00006.safetensors",
"gpt_neox.layers.19.attention.masked_bias": "model-00005-of-00006.safetensors",
"gpt_neox.layers.19.attention.query_key_value.bias": "model-00005-of-00006.safetensors",
"gpt_neox.layers.19.attention.query_key_value.weight": "model-00005-of-00006.safetensors",
"gpt_neox.layers.19.attention.rotary_emb.inv_freq": "model-00005-of-00006.safetensors",
"gpt_neox.layers.19.input_layernorm.bias": "model-00005-of-00006.safetensors",
"gpt_neox.layers.19.input_layernorm.weight": "model-00005-of-00006.safetensors",
"gpt_neox.layers.19.mlp.dense_4h_to_h.bias": "model-00005-of-00006.safetensors",
"gpt_neox.layers.19.mlp.dense_4h_to_h.weight": "model-00005-of-00006.safetensors",
"gpt_neox.layers.19.mlp.dense_h_to_4h.bias": "model-00005-of-00006.safetensors",
"gpt_neox.layers.19.mlp.dense_h_to_4h.weight": "model-00005-of-00006.safetensors",
"gpt_neox.layers.19.post_attention_layernorm.bias": "model-00005-of-00006.safetensors",
"gpt_neox.layers.19.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
"gpt_neox.layers.2.attention.bias": "model-00001-of-00006.safetensors",
"gpt_neox.layers.2.attention.dense.bias": "model-00001-of-00006.safetensors",
"gpt_neox.layers.2.attention.dense.weight": "model-00001-of-00006.safetensors",
"gpt_neox.layers.2.attention.masked_bias": "model-00001-of-00006.safetensors",
"gpt_neox.layers.2.attention.query_key_value.bias": "model-00001-of-00006.safetensors",
"gpt_neox.layers.2.attention.query_key_value.weight": "model-00001-of-00006.safetensors",
"gpt_neox.layers.2.attention.rotary_emb.inv_freq": "model-00001-of-00006.safetensors",
"gpt_neox.layers.2.input_layernorm.bias": "model-00001-of-00006.safetensors",
"gpt_neox.layers.2.input_layernorm.weight": "model-00001-of-00006.safetensors",
"gpt_neox.layers.2.mlp.dense_4h_to_h.bias": "model-00002-of-00006.safetensors",
"gpt_neox.layers.2.mlp.dense_4h_to_h.weight": "model-00002-of-00006.safetensors",
"gpt_neox.layers.2.mlp.dense_h_to_4h.bias": "model-00001-of-00006.safetensors",
"gpt_neox.layers.2.mlp.dense_h_to_4h.weight": "model-00001-of-00006.safetensors",
"gpt_neox.layers.2.post_attention_layernorm.bias": "model-00001-of-00006.safetensors",
"gpt_neox.layers.2.post_attention_layernorm.weight": "model-00001-of-00006.safetensors",
"gpt_neox.layers.20.attention.bias": "model-00005-of-00006.safetensors",
"gpt_neox.layers.20.attention.dense.bias": "model-00005-of-00006.safetensors",
"gpt_neox.layers.20.attention.dense.weight": "model-00005-of-00006.safetensors",
"gpt_neox.layers.20.attention.masked_bias": "model-00005-of-00006.safetensors",
"gpt_neox.layers.20.attention.query_key_value.bias": "model-00005-of-00006.safetensors",
"gpt_neox.layers.20.attention.query_key_value.weight": "model-00005-of-00006.safetensors",
"gpt_neox.layers.20.attention.rotary_emb.inv_freq": "model-00005-of-00006.safetensors",
"gpt_neox.layers.20.input_layernorm.bias": "model-00005-of-00006.safetensors",
"gpt_neox.layers.20.input_layernorm.weight": "model-00005-of-00006.safetensors",
"gpt_neox.layers.20.mlp.dense_4h_to_h.bias": "model-00005-of-00006.safetensors",
"gpt_neox.layers.20.mlp.dense_4h_to_h.weight": "model-00005-of-00006.safetensors",
"gpt_neox.layers.20.mlp.dense_h_to_4h.bias": "model-00005-of-00006.safetensors",
"gpt_neox.layers.20.mlp.dense_h_to_4h.weight": "model-00005-of-00006.safetensors",
"gpt_neox.layers.20.post_attention_layernorm.bias": "model-00005-of-00006.safetensors",
"gpt_neox.layers.20.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
"gpt_neox.layers.21.attention.bias": "model-00005-of-00006.safetensors",
"gpt_neox.layers.21.attention.dense.bias": "model-00005-of-00006.safetensors",
"gpt_neox.layers.21.attention.dense.weight": "model-00005-of-00006.safetensors",
"gpt_neox.layers.21.attention.masked_bias": "model-00005-of-00006.safetensors",
"gpt_neox.layers.21.attention.query_key_value.bias": "model-00005-of-00006.safetensors",
"gpt_neox.layers.21.attention.query_key_value.weight": "model-00005-of-00006.safetensors",
"gpt_neox.layers.21.attention.rotary_emb.inv_freq": "model-00005-of-00006.safetensors",
"gpt_neox.layers.21.input_layernorm.bias": "model-00005-of-00006.safetensors",
"gpt_neox.layers.21.input_layernorm.weight": "model-00005-of-00006.safetensors",
"gpt_neox.layers.21.mlp.dense_4h_to_h.bias": "model-00006-of-00006.safetensors",
"gpt_neox.layers.21.mlp.dense_4h_to_h.weight": "model-00006-of-00006.safetensors",
"gpt_neox.layers.21.mlp.dense_h_to_4h.bias": "model-00005-of-00006.safetensors",
"gpt_neox.layers.21.mlp.dense_h_to_4h.weight": "model-00005-of-00006.safetensors",
"gpt_neox.layers.21.post_attention_layernorm.bias": "model-00005-of-00006.safetensors",
"gpt_neox.layers.21.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
"gpt_neox.layers.22.attention.bias": "model-00006-of-00006.safetensors",
"gpt_neox.layers.22.attention.dense.bias": "model-00006-of-00006.safetensors",
"gpt_neox.layers.22.attention.dense.weight": "model-00006-of-00006.safetensors",
"gpt_neox.layers.22.attention.masked_bias": "model-00006-of-00006.safetensors",
"gpt_neox.layers.22.attention.query_key_value.bias": "model-00006-of-00006.safetensors",
"gpt_neox.layers.22.attention.query_key_value.weight": "model-00006-of-00006.safetensors",
"gpt_neox.layers.22.attention.rotary_emb.inv_freq": "model-00006-of-00006.safetensors",
"gpt_neox.layers.22.input_layernorm.bias": "model-00006-of-00006.safetensors",
"gpt_neox.layers.22.input_layernorm.weight": "model-00006-of-00006.safetensors",
"gpt_neox.layers.22.mlp.dense_4h_to_h.bias": "model-00006-of-00006.safetensors",
"gpt_neox.layers.22.mlp.dense_4h_to_h.weight": "model-00006-of-00006.safetensors",
"gpt_neox.layers.22.mlp.dense_h_to_4h.bias": "model-00006-of-00006.safetensors",
"gpt_neox.layers.22.mlp.dense_h_to_4h.weight": "model-00006-of-00006.safetensors",
"gpt_neox.layers.22.post_attention_layernorm.bias": "model-00006-of-00006.safetensors",
"gpt_neox.layers.22.post_attention_layernorm.weight": "model-00006-of-00006.safetensors",
"gpt_neox.layers.23.attention.bias": "model-00006-of-00006.safetensors",
"gpt_neox.layers.23.attention.dense.bias": "model-00006-of-00006.safetensors",
"gpt_neox.layers.23.attention.dense.weight": "model-00006-of-00006.safetensors",
"gpt_neox.layers.23.attention.masked_bias": "model-00006-of-00006.safetensors",
"gpt_neox.layers.23.attention.query_key_value.bias": "model-00006-of-00006.safetensors",
"gpt_neox.layers.23.attention.query_key_value.weight": "model-00006-of-00006.safetensors",
"gpt_neox.layers.23.attention.rotary_emb.inv_freq": "model-00006-of-00006.safetensors",
"gpt_neox.layers.23.input_layernorm.bias": "model-00006-of-00006.safetensors",
"gpt_neox.layers.23.input_layernorm.weight": "model-00006-of-00006.safetensors",
"gpt_neox.layers.23.mlp.dense_4h_to_h.bias": "model-00006-of-00006.safetensors",
"gpt_neox.layers.23.mlp.dense_4h_to_h.weight": "model-00006-of-00006.safetensors",
"gpt_neox.layers.23.mlp.dense_h_to_4h.bias": "model-00006-of-00006.safetensors",
"gpt_neox.layers.23.mlp.dense_h_to_4h.weight": "model-00006-of-00006.safetensors",
"gpt_neox.layers.23.post_attention_layernorm.bias": "model-00006-of-00006.safetensors",
"gpt_neox.layers.23.post_attention_layernorm.weight": "model-00006-of-00006.safetensors",
"gpt_neox.layers.3.attention.bias": "model-00002-of-00006.safetensors",
"gpt_neox.layers.3.attention.dense.bias": "model-00002-of-00006.safetensors",
"gpt_neox.layers.3.attention.dense.weight": "model-00002-of-00006.safetensors",
"gpt_neox.layers.3.attention.masked_bias": "model-00002-of-00006.safetensors",
"gpt_neox.layers.3.attention.query_key_value.bias": "model-00002-of-00006.safetensors",
"gpt_neox.layers.3.attention.query_key_value.weight": "model-00002-of-00006.safetensors",
"gpt_neox.layers.3.attention.rotary_emb.inv_freq": "model-00002-of-00006.safetensors",
"gpt_neox.layers.3.input_layernorm.bias": "model-00002-of-00006.safetensors",
"gpt_neox.layers.3.input_layernorm.weight": "model-00002-of-00006.safetensors",
"gpt_neox.layers.3.mlp.dense_4h_to_h.bias": "model-00002-of-00006.safetensors",
"gpt_neox.layers.3.mlp.dense_4h_to_h.weight": "model-00002-of-00006.safetensors",
"gpt_neox.layers.3.mlp.dense_h_to_4h.bias": "model-00002-of-00006.safetensors",
"gpt_neox.layers.3.mlp.dense_h_to_4h.weight": "model-00002-of-00006.safetensors",
"gpt_neox.layers.3.post_attention_layernorm.bias": "model-00002-of-00006.safetensors",
"gpt_neox.layers.3.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
"gpt_neox.layers.4.attention.bias": "model-00002-of-00006.safetensors",
"gpt_neox.layers.4.attention.dense.bias": "model-00002-of-00006.safetensors",
"gpt_neox.layers.4.attention.dense.weight": "model-00002-of-00006.safetensors",
"gpt_neox.layers.4.attention.masked_bias": "model-00002-of-00006.safetensors",
"gpt_neox.layers.4.attention.query_key_value.bias": "model-00002-of-00006.safetensors",
"gpt_neox.layers.4.attention.query_key_value.weight": "model-00002-of-00006.safetensors",
"gpt_neox.layers.4.attention.rotary_emb.inv_freq": "model-00002-of-00006.safetensors",
"gpt_neox.layers.4.input_layernorm.bias": "model-00002-of-00006.safetensors",
"gpt_neox.layers.4.input_layernorm.weight": "model-00002-of-00006.safetensors",
"gpt_neox.layers.4.mlp.dense_4h_to_h.bias": "model-00002-of-00006.safetensors",
"gpt_neox.layers.4.mlp.dense_4h_to_h.weight": "model-00002-of-00006.safetensors",
"gpt_neox.layers.4.mlp.dense_h_to_4h.bias": "model-00002-of-00006.safetensors",
"gpt_neox.layers.4.mlp.dense_h_to_4h.weight": "model-00002-of-00006.safetensors",
"gpt_neox.layers.4.post_attention_layernorm.bias": "model-00002-of-00006.safetensors",
"gpt_neox.layers.4.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
"gpt_neox.layers.5.attention.bias": "model-00002-of-00006.safetensors",
"gpt_neox.layers.5.attention.dense.bias": "model-00002-of-00006.safetensors",
"gpt_neox.layers.5.attention.dense.weight": "model-00002-of-00006.safetensors",
"gpt_neox.layers.5.attention.masked_bias": "model-00002-of-00006.safetensors",
"gpt_neox.layers.5.attention.query_key_value.bias": "model-00002-of-00006.safetensors",
"gpt_neox.layers.5.attention.query_key_value.weight": "model-00002-of-00006.safetensors",
"gpt_neox.layers.5.attention.rotary_emb.inv_freq": "model-00002-of-00006.safetensors",
"gpt_neox.layers.5.input_layernorm.bias": "model-00002-of-00006.safetensors",
"gpt_neox.layers.5.input_layernorm.weight": "model-00002-of-00006.safetensors",
"gpt_neox.layers.5.mlp.dense_4h_to_h.bias": "model-00002-of-00006.safetensors",
"gpt_neox.layers.5.mlp.dense_4h_to_h.weight": "model-00002-of-00006.safetensors",
"gpt_neox.layers.5.mlp.dense_h_to_4h.bias": "model-00002-of-00006.safetensors",
"gpt_neox.layers.5.mlp.dense_h_to_4h.weight": "model-00002-of-00006.safetensors",
"gpt_neox.layers.5.post_attention_layernorm.bias": "model-00002-of-00006.safetensors",
"gpt_neox.layers.5.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
"gpt_neox.layers.6.attention.bias": "model-00002-of-00006.safetensors",
"gpt_neox.layers.6.attention.dense.bias": "model-00002-of-00006.safetensors",
"gpt_neox.layers.6.attention.dense.weight": "model-00002-of-00006.safetensors",
"gpt_neox.layers.6.attention.masked_bias": "model-00002-of-00006.safetensors",
"gpt_neox.layers.6.attention.query_key_value.bias": "model-00002-of-00006.safetensors",
"gpt_neox.layers.6.attention.query_key_value.weight": "model-00002-of-00006.safetensors",
"gpt_neox.layers.6.attention.rotary_emb.inv_freq": "model-00002-of-00006.safetensors",
"gpt_neox.layers.6.input_layernorm.bias": "model-00002-of-00006.safetensors",
"gpt_neox.layers.6.input_layernorm.weight": "model-00002-of-00006.safetensors",
"gpt_neox.layers.6.mlp.dense_4h_to_h.bias": "model-00002-of-00006.safetensors",
"gpt_neox.layers.6.mlp.dense_4h_to_h.weight": "model-00002-of-00006.safetensors",
"gpt_neox.layers.6.mlp.dense_h_to_4h.bias": "model-00002-of-00006.safetensors",
"gpt_neox.layers.6.mlp.dense_h_to_4h.weight": "model-00002-of-00006.safetensors",
"gpt_neox.layers.6.post_attention_layernorm.bias": "model-00002-of-00006.safetensors",
"gpt_neox.layers.6.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
"gpt_neox.layers.7.attention.bias": "model-00002-of-00006.safetensors",
"gpt_neox.layers.7.attention.dense.bias": "model-00002-of-00006.safetensors",
"gpt_neox.layers.7.attention.dense.weight": "model-00002-of-00006.safetensors",
"gpt_neox.layers.7.attention.masked_bias": "model-00002-of-00006.safetensors",
"gpt_neox.layers.7.attention.query_key_value.bias": "model-00002-of-00006.safetensors",
"gpt_neox.layers.7.attention.query_key_value.weight": "model-00002-of-00006.safetensors",
"gpt_neox.layers.7.attention.rotary_emb.inv_freq": "model-00002-of-00006.safetensors",
"gpt_neox.layers.7.input_layernorm.bias": "model-00002-of-00006.safetensors",
"gpt_neox.layers.7.input_layernorm.weight": "model-00002-of-00006.safetensors",
"gpt_neox.layers.7.mlp.dense_4h_to_h.bias": "model-00003-of-00006.safetensors",
"gpt_neox.layers.7.mlp.dense_4h_to_h.weight": "model-00003-of-00006.safetensors",
"gpt_neox.layers.7.mlp.dense_h_to_4h.bias": "model-00003-of-00006.safetensors",
"gpt_neox.layers.7.mlp.dense_h_to_4h.weight": "model-00003-of-00006.safetensors",
"gpt_neox.layers.7.post_attention_layernorm.bias": "model-00002-of-00006.safetensors",
"gpt_neox.layers.7.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
"gpt_neox.layers.8.attention.bias": "model-00003-of-00006.safetensors",
"gpt_neox.layers.8.attention.dense.bias": "model-00003-of-00006.safetensors",
"gpt_neox.layers.8.attention.dense.weight": "model-00003-of-00006.safetensors",
"gpt_neox.layers.8.attention.masked_bias": "model-00003-of-00006.safetensors",
"gpt_neox.layers.8.attention.query_key_value.bias": "model-00003-of-00006.safetensors",
"gpt_neox.layers.8.attention.query_key_value.weight": "model-00003-of-00006.safetensors",
"gpt_neox.layers.8.attention.rotary_emb.inv_freq": "model-00003-of-00006.safetensors",
"gpt_neox.layers.8.input_layernorm.bias": "model-00003-of-00006.safetensors",
"gpt_neox.layers.8.input_layernorm.weight": "model-00003-of-00006.safetensors",
"gpt_neox.layers.8.mlp.dense_4h_to_h.bias": "model-00003-of-00006.safetensors",
"gpt_neox.layers.8.mlp.dense_4h_to_h.weight": "model-00003-of-00006.safetensors",
"gpt_neox.layers.8.mlp.dense_h_to_4h.bias": "model-00003-of-00006.safetensors",
"gpt_neox.layers.8.mlp.dense_h_to_4h.weight": "model-00003-of-00006.safetensors",
"gpt_neox.layers.8.post_attention_layernorm.bias": "model-00003-of-00006.safetensors",
"gpt_neox.layers.8.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
"gpt_neox.layers.9.attention.bias": "model-00003-of-00006.safetensors",
"gpt_neox.layers.9.attention.dense.bias": "model-00003-of-00006.safetensors",
"gpt_neox.layers.9.attention.dense.weight": "model-00003-of-00006.safetensors",
"gpt_neox.layers.9.attention.masked_bias": "model-00003-of-00006.safetensors",
"gpt_neox.layers.9.attention.query_key_value.bias": "model-00003-of-00006.safetensors",
"gpt_neox.layers.9.attention.query_key_value.weight": "model-00003-of-00006.safetensors",
"gpt_neox.layers.9.attention.rotary_emb.inv_freq": "model-00003-of-00006.safetensors",
"gpt_neox.layers.9.input_layernorm.bias": "model-00003-of-00006.safetensors",
"gpt_neox.layers.9.input_layernorm.weight": "model-00003-of-00006.safetensors",
"gpt_neox.layers.9.mlp.dense_4h_to_h.bias": "model-00003-of-00006.safetensors",
"gpt_neox.layers.9.mlp.dense_4h_to_h.weight": "model-00003-of-00006.safetensors",
"gpt_neox.layers.9.mlp.dense_h_to_4h.bias": "model-00003-of-00006.safetensors",
"gpt_neox.layers.9.mlp.dense_h_to_4h.weight": "model-00003-of-00006.safetensors",
"gpt_neox.layers.9.post_attention_layernorm.bias": "model-00003-of-00006.safetensors",
"gpt_neox.layers.9.post_attention_layernorm.weight": "model-00003-of-00006.safetensors"
}
}

3
pytorch_model-00001-of-00006.bin Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:3968ea358fbd9d987a6e0a4c197af26e89833dfb6787b9842ef54a26e56e1e36
size 487234535

3
pytorch_model-00002-of-00006.bin Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:79e25703874b887ca344d962847a6142ce03e978e4fcc5e2862976a59d8ae597
size 491008945

3
pytorch_model-00003-of-00006.bin Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:3319e70dc11c714162749ed186cf73bf61128ed86087b1194da27b3b13ac9640
size 516187121

3
pytorch_model-00004-of-00006.bin Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:a479543c3f602b028ca7f2e250e3436d2b10acff422e5141ae7cf7b4cb808179
size 499401713

3
pytorch_model-00005-of-00006.bin Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:02fda7c29dd4ee79d2822d39af39800a266dbb5915fc6d3a0d3ca58709b39ec4
size 486808179

3
pytorch_model-00006-of-00006.bin Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:8666e1b13b635a47535ca754803d5bb3209dbc67ec1eed5293b0baa3e8701e40
size 449445765

371
pytorch_model.bin.index.json Normal file
View File

@@ -0,0 +1,371 @@
{
"metadata": {
"total_size": 2841879344.0
},
"weight_map": {
"embed_out.weight": "pytorch_model-00006-of-00006.bin",
"gpt_neox.embed_in.weight": "pytorch_model-00001-of-00006.bin",
"gpt_neox.final_layer_norm.bias": "pytorch_model-00006-of-00006.bin",
"gpt_neox.final_layer_norm.weight": "pytorch_model-00006-of-00006.bin",
"gpt_neox.layers.0.attention.bias": "pytorch_model-00001-of-00006.bin",
"gpt_neox.layers.0.attention.dense.bias": "pytorch_model-00001-of-00006.bin",
"gpt_neox.layers.0.attention.dense.weight": "pytorch_model-00001-of-00006.bin",
"gpt_neox.layers.0.attention.masked_bias": "pytorch_model-00001-of-00006.bin",
"gpt_neox.layers.0.attention.query_key_value.bias": "pytorch_model-00001-of-00006.bin",
"gpt_neox.layers.0.attention.query_key_value.weight": "pytorch_model-00001-of-00006.bin",
"gpt_neox.layers.0.attention.rotary_emb.inv_freq": "pytorch_model-00001-of-00006.bin",
"gpt_neox.layers.0.input_layernorm.bias": "pytorch_model-00001-of-00006.bin",
"gpt_neox.layers.0.input_layernorm.weight": "pytorch_model-00001-of-00006.bin",
"gpt_neox.layers.0.mlp.dense_4h_to_h.bias": "pytorch_model-00001-of-00006.bin",
"gpt_neox.layers.0.mlp.dense_4h_to_h.weight": "pytorch_model-00001-of-00006.bin",
"gpt_neox.layers.0.mlp.dense_h_to_4h.bias": "pytorch_model-00001-of-00006.bin",
"gpt_neox.layers.0.mlp.dense_h_to_4h.weight": "pytorch_model-00001-of-00006.bin",
"gpt_neox.layers.0.post_attention_layernorm.bias": "pytorch_model-00001-of-00006.bin",
"gpt_neox.layers.0.post_attention_layernorm.weight": "pytorch_model-00001-of-00006.bin",
"gpt_neox.layers.1.attention.bias": "pytorch_model-00001-of-00006.bin",
"gpt_neox.layers.1.attention.dense.bias": "pytorch_model-00001-of-00006.bin",
"gpt_neox.layers.1.attention.dense.weight": "pytorch_model-00001-of-00006.bin",
"gpt_neox.layers.1.attention.masked_bias": "pytorch_model-00001-of-00006.bin",
"gpt_neox.layers.1.attention.query_key_value.bias": "pytorch_model-00001-of-00006.bin",
"gpt_neox.layers.1.attention.query_key_value.weight": "pytorch_model-00001-of-00006.bin",
"gpt_neox.layers.1.attention.rotary_emb.inv_freq": "pytorch_model-00001-of-00006.bin",
"gpt_neox.layers.1.input_layernorm.bias": "pytorch_model-00001-of-00006.bin",
"gpt_neox.layers.1.input_layernorm.weight": "pytorch_model-00001-of-00006.bin",
"gpt_neox.layers.1.mlp.dense_4h_to_h.bias": "pytorch_model-00001-of-00006.bin",
"gpt_neox.layers.1.mlp.dense_4h_to_h.weight": "pytorch_model-00001-of-00006.bin",
"gpt_neox.layers.1.mlp.dense_h_to_4h.bias": "pytorch_model-00001-of-00006.bin",
"gpt_neox.layers.1.mlp.dense_h_to_4h.weight": "pytorch_model-00001-of-00006.bin",
"gpt_neox.layers.1.post_attention_layernorm.bias": "pytorch_model-00001-of-00006.bin",
"gpt_neox.layers.1.post_attention_layernorm.weight": "pytorch_model-00001-of-00006.bin",
"gpt_neox.layers.10.attention.bias": "pytorch_model-00003-of-00006.bin",
"gpt_neox.layers.10.attention.dense.bias": "pytorch_model-00003-of-00006.bin",
"gpt_neox.layers.10.attention.dense.weight": "pytorch_model-00003-of-00006.bin",
"gpt_neox.layers.10.attention.masked_bias": "pytorch_model-00003-of-00006.bin",
"gpt_neox.layers.10.attention.query_key_value.bias": "pytorch_model-00003-of-00006.bin",
"gpt_neox.layers.10.attention.query_key_value.weight": "pytorch_model-00003-of-00006.bin",
"gpt_neox.layers.10.attention.rotary_emb.inv_freq": "pytorch_model-00003-of-00006.bin",
"gpt_neox.layers.10.input_layernorm.bias": "pytorch_model-00003-of-00006.bin",
"gpt_neox.layers.10.input_layernorm.weight": "pytorch_model-00003-of-00006.bin",
"gpt_neox.layers.10.mlp.dense_4h_to_h.bias": "pytorch_model-00003-of-00006.bin",
"gpt_neox.layers.10.mlp.dense_4h_to_h.weight": "pytorch_model-00003-of-00006.bin",
"gpt_neox.layers.10.mlp.dense_h_to_4h.bias": "pytorch_model-00003-of-00006.bin",
"gpt_neox.layers.10.mlp.dense_h_to_4h.weight": "pytorch_model-00003-of-00006.bin",
"gpt_neox.layers.10.post_attention_layernorm.bias": "pytorch_model-00003-of-00006.bin",
"gpt_neox.layers.10.post_attention_layernorm.weight": "pytorch_model-00003-of-00006.bin",
"gpt_neox.layers.11.attention.bias": "pytorch_model-00003-of-00006.bin",
"gpt_neox.layers.11.attention.dense.bias": "pytorch_model-00003-of-00006.bin",
"gpt_neox.layers.11.attention.dense.weight": "pytorch_model-00003-of-00006.bin",
"gpt_neox.layers.11.attention.masked_bias": "pytorch_model-00003-of-00006.bin",
"gpt_neox.layers.11.attention.query_key_value.bias": "pytorch_model-00003-of-00006.bin",
"gpt_neox.layers.11.attention.query_key_value.weight": "pytorch_model-00003-of-00006.bin",
"gpt_neox.layers.11.attention.rotary_emb.inv_freq": "pytorch_model-00003-of-00006.bin",
"gpt_neox.layers.11.input_layernorm.bias": "pytorch_model-00003-of-00006.bin",
"gpt_neox.layers.11.input_layernorm.weight": "pytorch_model-00003-of-00006.bin",
"gpt_neox.layers.11.mlp.dense_4h_to_h.bias": "pytorch_model-00003-of-00006.bin",
"gpt_neox.layers.11.mlp.dense_4h_to_h.weight": "pytorch_model-00003-of-00006.bin",
"gpt_neox.layers.11.mlp.dense_h_to_4h.bias": "pytorch_model-00003-of-00006.bin",
"gpt_neox.layers.11.mlp.dense_h_to_4h.weight": "pytorch_model-00003-of-00006.bin",
"gpt_neox.layers.11.post_attention_layernorm.bias": "pytorch_model-00003-of-00006.bin",
"gpt_neox.layers.11.post_attention_layernorm.weight": "pytorch_model-00003-of-00006.bin",
"gpt_neox.layers.12.attention.bias": "pytorch_model-00003-of-00006.bin",
"gpt_neox.layers.12.attention.dense.bias": "pytorch_model-00004-of-00006.bin",
"gpt_neox.layers.12.attention.dense.weight": "pytorch_model-00004-of-00006.bin",
"gpt_neox.layers.12.attention.masked_bias": "pytorch_model-00003-of-00006.bin",
"gpt_neox.layers.12.attention.query_key_value.bias": "pytorch_model-00003-of-00006.bin",
"gpt_neox.layers.12.attention.query_key_value.weight": "pytorch_model-00003-of-00006.bin",
"gpt_neox.layers.12.attention.rotary_emb.inv_freq": "pytorch_model-00003-of-00006.bin",
"gpt_neox.layers.12.input_layernorm.bias": "pytorch_model-00003-of-00006.bin",
"gpt_neox.layers.12.input_layernorm.weight": "pytorch_model-00003-of-00006.bin",
"gpt_neox.layers.12.mlp.dense_4h_to_h.bias": "pytorch_model-00004-of-00006.bin",
"gpt_neox.layers.12.mlp.dense_4h_to_h.weight": "pytorch_model-00004-of-00006.bin",
"gpt_neox.layers.12.mlp.dense_h_to_4h.bias": "pytorch_model-00004-of-00006.bin",
"gpt_neox.layers.12.mlp.dense_h_to_4h.weight": "pytorch_model-00004-of-00006.bin",
"gpt_neox.layers.12.post_attention_layernorm.bias": "pytorch_model-00003-of-00006.bin",
"gpt_neox.layers.12.post_attention_layernorm.weight": "pytorch_model-00003-of-00006.bin",
"gpt_neox.layers.13.attention.bias": "pytorch_model-00004-of-00006.bin",
"gpt_neox.layers.13.attention.dense.bias": "pytorch_model-00004-of-00006.bin",
"gpt_neox.layers.13.attention.dense.weight": "pytorch_model-00004-of-00006.bin",
"gpt_neox.layers.13.attention.masked_bias": "pytorch_model-00004-of-00006.bin",
"gpt_neox.layers.13.attention.query_key_value.bias": "pytorch_model-00004-of-00006.bin",
"gpt_neox.layers.13.attention.query_key_value.weight": "pytorch_model-00004-of-00006.bin",
"gpt_neox.layers.13.attention.rotary_emb.inv_freq": "pytorch_model-00004-of-00006.bin",
"gpt_neox.layers.13.input_layernorm.bias": "pytorch_model-00004-of-00006.bin",
"gpt_neox.layers.13.input_layernorm.weight": "pytorch_model-00004-of-00006.bin",
"gpt_neox.layers.13.mlp.dense_4h_to_h.bias": "pytorch_model-00004-of-00006.bin",
"gpt_neox.layers.13.mlp.dense_4h_to_h.weight": "pytorch_model-00004-of-00006.bin",
"gpt_neox.layers.13.mlp.dense_h_to_4h.bias": "pytorch_model-00004-of-00006.bin",
"gpt_neox.layers.13.mlp.dense_h_to_4h.weight": "pytorch_model-00004-of-00006.bin",
"gpt_neox.layers.13.post_attention_layernorm.bias": "pytorch_model-00004-of-00006.bin",
"gpt_neox.layers.13.post_attention_layernorm.weight": "pytorch_model-00004-of-00006.bin",
"gpt_neox.layers.14.attention.bias": "pytorch_model-00004-of-00006.bin",
"gpt_neox.layers.14.attention.dense.bias": "pytorch_model-00004-of-00006.bin",
"gpt_neox.layers.14.attention.dense.weight": "pytorch_model-00004-of-00006.bin",
"gpt_neox.layers.14.attention.masked_bias": "pytorch_model-00004-of-00006.bin",
"gpt_neox.layers.14.attention.query_key_value.bias": "pytorch_model-00004-of-00006.bin",
"gpt_neox.layers.14.attention.query_key_value.weight": "pytorch_model-00004-of-00006.bin",
"gpt_neox.layers.14.attention.rotary_emb.inv_freq": "pytorch_model-00004-of-00006.bin",
"gpt_neox.layers.14.input_layernorm.bias": "pytorch_model-00004-of-00006.bin",
"gpt_neox.layers.14.input_layernorm.weight": "pytorch_model-00004-of-00006.bin",
"gpt_neox.layers.14.mlp.dense_4h_to_h.bias": "pytorch_model-00004-of-00006.bin",
"gpt_neox.layers.14.mlp.dense_4h_to_h.weight": "pytorch_model-00004-of-00006.bin",
"gpt_neox.layers.14.mlp.dense_h_to_4h.bias": "pytorch_model-00004-of-00006.bin",
"gpt_neox.layers.14.mlp.dense_h_to_4h.weight": "pytorch_model-00004-of-00006.bin",
"gpt_neox.layers.14.post_attention_layernorm.bias": "pytorch_model-00004-of-00006.bin",
"gpt_neox.layers.14.post_attention_layernorm.weight": "pytorch_model-00004-of-00006.bin",
"gpt_neox.layers.15.attention.bias": "pytorch_model-00004-of-00006.bin",
"gpt_neox.layers.15.attention.dense.bias": "pytorch_model-00004-of-00006.bin",
"gpt_neox.layers.15.attention.dense.weight": "pytorch_model-00004-of-00006.bin",
"gpt_neox.layers.15.attention.masked_bias": "pytorch_model-00004-of-00006.bin",
"gpt_neox.layers.15.attention.query_key_value.bias": "pytorch_model-00004-of-00006.bin",
"gpt_neox.layers.15.attention.query_key_value.weight": "pytorch_model-00004-of-00006.bin",
"gpt_neox.layers.15.attention.rotary_emb.inv_freq": "pytorch_model-00004-of-00006.bin",
"gpt_neox.layers.15.input_layernorm.bias": "pytorch_model-00004-of-00006.bin",
"gpt_neox.layers.15.input_layernorm.weight": "pytorch_model-00004-of-00006.bin",
"gpt_neox.layers.15.mlp.dense_4h_to_h.bias": "pytorch_model-00004-of-00006.bin",
"gpt_neox.layers.15.mlp.dense_4h_to_h.weight": "pytorch_model-00004-of-00006.bin",
"gpt_neox.layers.15.mlp.dense_h_to_4h.bias": "pytorch_model-00004-of-00006.bin",
"gpt_neox.layers.15.mlp.dense_h_to_4h.weight": "pytorch_model-00004-of-00006.bin",
"gpt_neox.layers.15.post_attention_layernorm.bias": "pytorch_model-00004-of-00006.bin",
"gpt_neox.layers.15.post_attention_layernorm.weight": "pytorch_model-00004-of-00006.bin",
"gpt_neox.layers.16.attention.bias": "pytorch_model-00004-of-00006.bin",
"gpt_neox.layers.16.attention.dense.bias": "pytorch_model-00004-of-00006.bin",
"gpt_neox.layers.16.attention.dense.weight": "pytorch_model-00004-of-00006.bin",
"gpt_neox.layers.16.attention.masked_bias": "pytorch_model-00004-of-00006.bin",
"gpt_neox.layers.16.attention.query_key_value.bias": "pytorch_model-00004-of-00006.bin",
"gpt_neox.layers.16.attention.query_key_value.weight": "pytorch_model-00004-of-00006.bin",
"gpt_neox.layers.16.attention.rotary_emb.inv_freq": "pytorch_model-00004-of-00006.bin",
"gpt_neox.layers.16.input_layernorm.bias": "pytorch_model-00004-of-00006.bin",
"gpt_neox.layers.16.input_layernorm.weight": "pytorch_model-00004-of-00006.bin",
"gpt_neox.layers.16.mlp.dense_4h_to_h.bias": "pytorch_model-00004-of-00006.bin",
"gpt_neox.layers.16.mlp.dense_4h_to_h.weight": "pytorch_model-00004-of-00006.bin",
"gpt_neox.layers.16.mlp.dense_h_to_4h.bias": "pytorch_model-00004-of-00006.bin",
"gpt_neox.layers.16.mlp.dense_h_to_4h.weight": "pytorch_model-00004-of-00006.bin",
"gpt_neox.layers.16.post_attention_layernorm.bias": "pytorch_model-00004-of-00006.bin",
"gpt_neox.layers.16.post_attention_layernorm.weight": "pytorch_model-00004-of-00006.bin",
"gpt_neox.layers.17.attention.bias": "pytorch_model-00004-of-00006.bin",
"gpt_neox.layers.17.attention.dense.bias": "pytorch_model-00005-of-00006.bin",
"gpt_neox.layers.17.attention.dense.weight": "pytorch_model-00005-of-00006.bin",
"gpt_neox.layers.17.attention.masked_bias": "pytorch_model-00004-of-00006.bin",
"gpt_neox.layers.17.attention.query_key_value.bias": "pytorch_model-00005-of-00006.bin",
"gpt_neox.layers.17.attention.query_key_value.weight": "pytorch_model-00005-of-00006.bin",
"gpt_neox.layers.17.attention.rotary_emb.inv_freq": "pytorch_model-00004-of-00006.bin",
"gpt_neox.layers.17.input_layernorm.bias": "pytorch_model-00004-of-00006.bin",
"gpt_neox.layers.17.input_layernorm.weight": "pytorch_model-00004-of-00006.bin",
"gpt_neox.layers.17.mlp.dense_4h_to_h.bias": "pytorch_model-00005-of-00006.bin",
"gpt_neox.layers.17.mlp.dense_4h_to_h.weight": "pytorch_model-00005-of-00006.bin",
"gpt_neox.layers.17.mlp.dense_h_to_4h.bias": "pytorch_model-00005-of-00006.bin",
"gpt_neox.layers.17.mlp.dense_h_to_4h.weight": "pytorch_model-00005-of-00006.bin",
"gpt_neox.layers.17.post_attention_layernorm.bias": "pytorch_model-00004-of-00006.bin",
"gpt_neox.layers.17.post_attention_layernorm.weight": "pytorch_model-00004-of-00006.bin",
"gpt_neox.layers.18.attention.bias": "pytorch_model-00005-of-00006.bin",
"gpt_neox.layers.18.attention.dense.bias": "pytorch_model-00005-of-00006.bin",
"gpt_neox.layers.18.attention.dense.weight": "pytorch_model-00005-of-00006.bin",
"gpt_neox.layers.18.attention.masked_bias": "pytorch_model-00005-of-00006.bin",
"gpt_neox.layers.18.attention.query_key_value.bias": "pytorch_model-00005-of-00006.bin",
"gpt_neox.layers.18.attention.query_key_value.weight": "pytorch_model-00005-of-00006.bin",
"gpt_neox.layers.18.attention.rotary_emb.inv_freq": "pytorch_model-00005-of-00006.bin",
"gpt_neox.layers.18.input_layernorm.bias": "pytorch_model-00005-of-00006.bin",
"gpt_neox.layers.18.input_layernorm.weight": "pytorch_model-00005-of-00006.bin",
"gpt_neox.layers.18.mlp.dense_4h_to_h.bias": "pytorch_model-00005-of-00006.bin",
"gpt_neox.layers.18.mlp.dense_4h_to_h.weight": "pytorch_model-00005-of-00006.bin",
"gpt_neox.layers.18.mlp.dense_h_to_4h.bias": "pytorch_model-00005-of-00006.bin",
"gpt_neox.layers.18.mlp.dense_h_to_4h.weight": "pytorch_model-00005-of-00006.bin",
"gpt_neox.layers.18.post_attention_layernorm.bias": "pytorch_model-00005-of-00006.bin",
"gpt_neox.layers.18.post_attention_layernorm.weight": "pytorch_model-00005-of-00006.bin",
"gpt_neox.layers.19.attention.bias": "pytorch_model-00005-of-00006.bin",
"gpt_neox.layers.19.attention.dense.bias": "pytorch_model-00005-of-00006.bin",
"gpt_neox.layers.19.attention.dense.weight": "pytorch_model-00005-of-00006.bin",
"gpt_neox.layers.19.attention.masked_bias": "pytorch_model-00005-of-00006.bin",
"gpt_neox.layers.19.attention.query_key_value.bias": "pytorch_model-00005-of-00006.bin",
"gpt_neox.layers.19.attention.query_key_value.weight": "pytorch_model-00005-of-00006.bin",
"gpt_neox.layers.19.attention.rotary_emb.inv_freq": "pytorch_model-00005-of-00006.bin",
"gpt_neox.layers.19.input_layernorm.bias": "pytorch_model-00005-of-00006.bin",
"gpt_neox.layers.19.input_layernorm.weight": "pytorch_model-00005-of-00006.bin",
"gpt_neox.layers.19.mlp.dense_4h_to_h.bias": "pytorch_model-00005-of-00006.bin",
"gpt_neox.layers.19.mlp.dense_4h_to_h.weight": "pytorch_model-00005-of-00006.bin",
"gpt_neox.layers.19.mlp.dense_h_to_4h.bias": "pytorch_model-00005-of-00006.bin",
"gpt_neox.layers.19.mlp.dense_h_to_4h.weight": "pytorch_model-00005-of-00006.bin",
"gpt_neox.layers.19.post_attention_layernorm.bias": "pytorch_model-00005-of-00006.bin",
"gpt_neox.layers.19.post_attention_layernorm.weight": "pytorch_model-00005-of-00006.bin",
"gpt_neox.layers.2.attention.bias": "pytorch_model-00001-of-00006.bin",
"gpt_neox.layers.2.attention.dense.bias": "pytorch_model-00001-of-00006.bin",
"gpt_neox.layers.2.attention.dense.weight": "pytorch_model-00001-of-00006.bin",
"gpt_neox.layers.2.attention.masked_bias": "pytorch_model-00001-of-00006.bin",
"gpt_neox.layers.2.attention.query_key_value.bias": "pytorch_model-00001-of-00006.bin",
"gpt_neox.layers.2.attention.query_key_value.weight": "pytorch_model-00001-of-00006.bin",
"gpt_neox.layers.2.attention.rotary_emb.inv_freq": "pytorch_model-00001-of-00006.bin",
"gpt_neox.layers.2.input_layernorm.bias": "pytorch_model-00001-of-00006.bin",
"gpt_neox.layers.2.input_layernorm.weight": "pytorch_model-00001-of-00006.bin",
"gpt_neox.layers.2.mlp.dense_4h_to_h.bias": "pytorch_model-00002-of-00006.bin",
"gpt_neox.layers.2.mlp.dense_4h_to_h.weight": "pytorch_model-00002-of-00006.bin",
"gpt_neox.layers.2.mlp.dense_h_to_4h.bias": "pytorch_model-00001-of-00006.bin",
"gpt_neox.layers.2.mlp.dense_h_to_4h.weight": "pytorch_model-00001-of-00006.bin",
"gpt_neox.layers.2.post_attention_layernorm.bias": "pytorch_model-00001-of-00006.bin",
"gpt_neox.layers.2.post_attention_layernorm.weight": "pytorch_model-00001-of-00006.bin",
"gpt_neox.layers.20.attention.bias": "pytorch_model-00005-of-00006.bin",
"gpt_neox.layers.20.attention.dense.bias": "pytorch_model-00005-of-00006.bin",
"gpt_neox.layers.20.attention.dense.weight": "pytorch_model-00005-of-00006.bin",
"gpt_neox.layers.20.attention.masked_bias": "pytorch_model-00005-of-00006.bin",
"gpt_neox.layers.20.attention.query_key_value.bias": "pytorch_model-00005-of-00006.bin",
"gpt_neox.layers.20.attention.query_key_value.weight": "pytorch_model-00005-of-00006.bin",
"gpt_neox.layers.20.attention.rotary_emb.inv_freq": "pytorch_model-00005-of-00006.bin",
"gpt_neox.layers.20.input_layernorm.bias": "pytorch_model-00005-of-00006.bin",
"gpt_neox.layers.20.input_layernorm.weight": "pytorch_model-00005-of-00006.bin",
"gpt_neox.layers.20.mlp.dense_4h_to_h.bias": "pytorch_model-00005-of-00006.bin",
"gpt_neox.layers.20.mlp.dense_4h_to_h.weight": "pytorch_model-00005-of-00006.bin",
"gpt_neox.layers.20.mlp.dense_h_to_4h.bias": "pytorch_model-00005-of-00006.bin",
"gpt_neox.layers.20.mlp.dense_h_to_4h.weight": "pytorch_model-00005-of-00006.bin",
"gpt_neox.layers.20.post_attention_layernorm.bias": "pytorch_model-00005-of-00006.bin",
"gpt_neox.layers.20.post_attention_layernorm.weight": "pytorch_model-00005-of-00006.bin",
"gpt_neox.layers.21.attention.bias": "pytorch_model-00005-of-00006.bin",
"gpt_neox.layers.21.attention.dense.bias": "pytorch_model-00005-of-00006.bin",
"gpt_neox.layers.21.attention.dense.weight": "pytorch_model-00005-of-00006.bin",
"gpt_neox.layers.21.attention.masked_bias": "pytorch_model-00005-of-00006.bin",
"gpt_neox.layers.21.attention.query_key_value.bias": "pytorch_model-00005-of-00006.bin",
"gpt_neox.layers.21.attention.query_key_value.weight": "pytorch_model-00005-of-00006.bin",
"gpt_neox.layers.21.attention.rotary_emb.inv_freq": "pytorch_model-00005-of-00006.bin",
"gpt_neox.layers.21.input_layernorm.bias": "pytorch_model-00005-of-00006.bin",
"gpt_neox.layers.21.input_layernorm.weight": "pytorch_model-00005-of-00006.bin",
"gpt_neox.layers.21.mlp.dense_4h_to_h.bias": "pytorch_model-00006-of-00006.bin",
"gpt_neox.layers.21.mlp.dense_4h_to_h.weight": "pytorch_model-00006-of-00006.bin",
"gpt_neox.layers.21.mlp.dense_h_to_4h.bias": "pytorch_model-00005-of-00006.bin",
"gpt_neox.layers.21.mlp.dense_h_to_4h.weight": "pytorch_model-00005-of-00006.bin",
"gpt_neox.layers.21.post_attention_layernorm.bias": "pytorch_model-00005-of-00006.bin",
"gpt_neox.layers.21.post_attention_layernorm.weight": "pytorch_model-00005-of-00006.bin",
"gpt_neox.layers.22.attention.bias": "pytorch_model-00006-of-00006.bin",
"gpt_neox.layers.22.attention.dense.bias": "pytorch_model-00006-of-00006.bin",
"gpt_neox.layers.22.attention.dense.weight": "pytorch_model-00006-of-00006.bin",
"gpt_neox.layers.22.attention.masked_bias": "pytorch_model-00006-of-00006.bin",
"gpt_neox.layers.22.attention.query_key_value.bias": "pytorch_model-00006-of-00006.bin",
"gpt_neox.layers.22.attention.query_key_value.weight": "pytorch_model-00006-of-00006.bin",
"gpt_neox.layers.22.attention.rotary_emb.inv_freq": "pytorch_model-00006-of-00006.bin",
"gpt_neox.layers.22.input_layernorm.bias": "pytorch_model-00006-of-00006.bin",
"gpt_neox.layers.22.input_layernorm.weight": "pytorch_model-00006-of-00006.bin",
"gpt_neox.layers.22.mlp.dense_4h_to_h.bias": "pytorch_model-00006-of-00006.bin",
"gpt_neox.layers.22.mlp.dense_4h_to_h.weight": "pytorch_model-00006-of-00006.bin",
"gpt_neox.layers.22.mlp.dense_h_to_4h.bias": "pytorch_model-00006-of-00006.bin",
"gpt_neox.layers.22.mlp.dense_h_to_4h.weight": "pytorch_model-00006-of-00006.bin",
"gpt_neox.layers.22.post_attention_layernorm.bias": "pytorch_model-00006-of-00006.bin",
"gpt_neox.layers.22.post_attention_layernorm.weight": "pytorch_model-00006-of-00006.bin",
"gpt_neox.layers.23.attention.bias": "pytorch_model-00006-of-00006.bin",
"gpt_neox.layers.23.attention.dense.bias": "pytorch_model-00006-of-00006.bin",
"gpt_neox.layers.23.attention.dense.weight": "pytorch_model-00006-of-00006.bin",
"gpt_neox.layers.23.attention.masked_bias": "pytorch_model-00006-of-00006.bin",
"gpt_neox.layers.23.attention.query_key_value.bias": "pytorch_model-00006-of-00006.bin",
"gpt_neox.layers.23.attention.query_key_value.weight": "pytorch_model-00006-of-00006.bin",
"gpt_neox.layers.23.attention.rotary_emb.inv_freq": "pytorch_model-00006-of-00006.bin",
"gpt_neox.layers.23.input_layernorm.bias": "pytorch_model-00006-of-00006.bin",
"gpt_neox.layers.23.input_layernorm.weight": "pytorch_model-00006-of-00006.bin",
"gpt_neox.layers.23.mlp.dense_4h_to_h.bias": "pytorch_model-00006-of-00006.bin",
"gpt_neox.layers.23.mlp.dense_4h_to_h.weight": "pytorch_model-00006-of-00006.bin",
"gpt_neox.layers.23.mlp.dense_h_to_4h.bias": "pytorch_model-00006-of-00006.bin",
"gpt_neox.layers.23.mlp.dense_h_to_4h.weight": "pytorch_model-00006-of-00006.bin",
"gpt_neox.layers.23.post_attention_layernorm.bias": "pytorch_model-00006-of-00006.bin",
"gpt_neox.layers.23.post_attention_layernorm.weight": "pytorch_model-00006-of-00006.bin",
"gpt_neox.layers.3.attention.bias": "pytorch_model-00002-of-00006.bin",
"gpt_neox.layers.3.attention.dense.bias": "pytorch_model-00002-of-00006.bin",
"gpt_neox.layers.3.attention.dense.weight": "pytorch_model-00002-of-00006.bin",
"gpt_neox.layers.3.attention.masked_bias": "pytorch_model-00002-of-00006.bin",
"gpt_neox.layers.3.attention.query_key_value.bias": "pytorch_model-00002-of-00006.bin",
"gpt_neox.layers.3.attention.query_key_value.weight": "pytorch_model-00002-of-00006.bin",
"gpt_neox.layers.3.attention.rotary_emb.inv_freq": "pytorch_model-00002-of-00006.bin",
"gpt_neox.layers.3.input_layernorm.bias": "pytorch_model-00002-of-00006.bin",
"gpt_neox.layers.3.input_layernorm.weight": "pytorch_model-00002-of-00006.bin",
"gpt_neox.layers.3.mlp.dense_4h_to_h.bias": "pytorch_model-00002-of-00006.bin",
"gpt_neox.layers.3.mlp.dense_4h_to_h.weight": "pytorch_model-00002-of-00006.bin",
"gpt_neox.layers.3.mlp.dense_h_to_4h.bias": "pytorch_model-00002-of-00006.bin",
"gpt_neox.layers.3.mlp.dense_h_to_4h.weight": "pytorch_model-00002-of-00006.bin",
"gpt_neox.layers.3.post_attention_layernorm.bias": "pytorch_model-00002-of-00006.bin",
"gpt_neox.layers.3.post_attention_layernorm.weight": "pytorch_model-00002-of-00006.bin",
"gpt_neox.layers.4.attention.bias": "pytorch_model-00002-of-00006.bin",
"gpt_neox.layers.4.attention.dense.bias": "pytorch_model-00002-of-00006.bin",
"gpt_neox.layers.4.attention.dense.weight": "pytorch_model-00002-of-00006.bin",
"gpt_neox.layers.4.attention.masked_bias": "pytorch_model-00002-of-00006.bin",
"gpt_neox.layers.4.attention.query_key_value.bias": "pytorch_model-00002-of-00006.bin",
"gpt_neox.layers.4.attention.query_key_value.weight": "pytorch_model-00002-of-00006.bin",
"gpt_neox.layers.4.attention.rotary_emb.inv_freq": "pytorch_model-00002-of-00006.bin",
"gpt_neox.layers.4.input_layernorm.bias": "pytorch_model-00002-of-00006.bin",
"gpt_neox.layers.4.input_layernorm.weight": "pytorch_model-00002-of-00006.bin",
"gpt_neox.layers.4.mlp.dense_4h_to_h.bias": "pytorch_model-00002-of-00006.bin",
"gpt_neox.layers.4.mlp.dense_4h_to_h.weight": "pytorch_model-00002-of-00006.bin",
"gpt_neox.layers.4.mlp.dense_h_to_4h.bias": "pytorch_model-00002-of-00006.bin",
"gpt_neox.layers.4.mlp.dense_h_to_4h.weight": "pytorch_model-00002-of-00006.bin",
"gpt_neox.layers.4.post_attention_layernorm.bias": "pytorch_model-00002-of-00006.bin",
"gpt_neox.layers.4.post_attention_layernorm.weight": "pytorch_model-00002-of-00006.bin",
"gpt_neox.layers.5.attention.bias": "pytorch_model-00002-of-00006.bin",
"gpt_neox.layers.5.attention.dense.bias": "pytorch_model-00002-of-00006.bin",
"gpt_neox.layers.5.attention.dense.weight": "pytorch_model-00002-of-00006.bin",
"gpt_neox.layers.5.attention.masked_bias": "pytorch_model-00002-of-00006.bin",
"gpt_neox.layers.5.attention.query_key_value.bias": "pytorch_model-00002-of-00006.bin",
"gpt_neox.layers.5.attention.query_key_value.weight": "pytorch_model-00002-of-00006.bin",
"gpt_neox.layers.5.attention.rotary_emb.inv_freq": "pytorch_model-00002-of-00006.bin",
"gpt_neox.layers.5.input_layernorm.bias": "pytorch_model-00002-of-00006.bin",
"gpt_neox.layers.5.input_layernorm.weight": "pytorch_model-00002-of-00006.bin",
"gpt_neox.layers.5.mlp.dense_4h_to_h.bias": "pytorch_model-00002-of-00006.bin",
"gpt_neox.layers.5.mlp.dense_4h_to_h.weight": "pytorch_model-00002-of-00006.bin",
"gpt_neox.layers.5.mlp.dense_h_to_4h.bias": "pytorch_model-00002-of-00006.bin",
"gpt_neox.layers.5.mlp.dense_h_to_4h.weight": "pytorch_model-00002-of-00006.bin",
"gpt_neox.layers.5.post_attention_layernorm.bias": "pytorch_model-00002-of-00006.bin",
"gpt_neox.layers.5.post_attention_layernorm.weight": "pytorch_model-00002-of-00006.bin",
"gpt_neox.layers.6.attention.bias": "pytorch_model-00002-of-00006.bin",
"gpt_neox.layers.6.attention.dense.bias": "pytorch_model-00002-of-00006.bin",
"gpt_neox.layers.6.attention.dense.weight": "pytorch_model-00002-of-00006.bin",
"gpt_neox.layers.6.attention.masked_bias": "pytorch_model-00002-of-00006.bin",
"gpt_neox.layers.6.attention.query_key_value.bias": "pytorch_model-00002-of-00006.bin",
"gpt_neox.layers.6.attention.query_key_value.weight": "pytorch_model-00002-of-00006.bin",
"gpt_neox.layers.6.attention.rotary_emb.inv_freq": "pytorch_model-00002-of-00006.bin",
"gpt_neox.layers.6.input_layernorm.bias": "pytorch_model-00002-of-00006.bin",
"gpt_neox.layers.6.input_layernorm.weight": "pytorch_model-00002-of-00006.bin",
"gpt_neox.layers.6.mlp.dense_4h_to_h.bias": "pytorch_model-00002-of-00006.bin",
"gpt_neox.layers.6.mlp.dense_4h_to_h.weight": "pytorch_model-00002-of-00006.bin",
"gpt_neox.layers.6.mlp.dense_h_to_4h.bias": "pytorch_model-00002-of-00006.bin",
"gpt_neox.layers.6.mlp.dense_h_to_4h.weight": "pytorch_model-00002-of-00006.bin",
"gpt_neox.layers.6.post_attention_layernorm.bias": "pytorch_model-00002-of-00006.bin",
"gpt_neox.layers.6.post_attention_layernorm.weight": "pytorch_model-00002-of-00006.bin",
"gpt_neox.layers.7.attention.bias": "pytorch_model-00002-of-00006.bin",
"gpt_neox.layers.7.attention.dense.bias": "pytorch_model-00002-of-00006.bin",
"gpt_neox.layers.7.attention.dense.weight": "pytorch_model-00002-of-00006.bin",
"gpt_neox.layers.7.attention.masked_bias": "pytorch_model-00002-of-00006.bin",
"gpt_neox.layers.7.attention.query_key_value.bias": "pytorch_model-00002-of-00006.bin",
"gpt_neox.layers.7.attention.query_key_value.weight": "pytorch_model-00002-of-00006.bin",
"gpt_neox.layers.7.attention.rotary_emb.inv_freq": "pytorch_model-00002-of-00006.bin",
"gpt_neox.layers.7.input_layernorm.bias": "pytorch_model-00002-of-00006.bin",
"gpt_neox.layers.7.input_layernorm.weight": "pytorch_model-00002-of-00006.bin",
"gpt_neox.layers.7.mlp.dense_4h_to_h.bias": "pytorch_model-00003-of-00006.bin",
"gpt_neox.layers.7.mlp.dense_4h_to_h.weight": "pytorch_model-00003-of-00006.bin",
"gpt_neox.layers.7.mlp.dense_h_to_4h.bias": "pytorch_model-00003-of-00006.bin",
"gpt_neox.layers.7.mlp.dense_h_to_4h.weight": "pytorch_model-00003-of-00006.bin",
"gpt_neox.layers.7.post_attention_layernorm.bias": "pytorch_model-00002-of-00006.bin",
"gpt_neox.layers.7.post_attention_layernorm.weight": "pytorch_model-00002-of-00006.bin",
"gpt_neox.layers.8.attention.bias": "pytorch_model-00003-of-00006.bin",
"gpt_neox.layers.8.attention.dense.bias": "pytorch_model-00003-of-00006.bin",
"gpt_neox.layers.8.attention.dense.weight": "pytorch_model-00003-of-00006.bin",
"gpt_neox.layers.8.attention.masked_bias": "pytorch_model-00003-of-00006.bin",
"gpt_neox.layers.8.attention.query_key_value.bias": "pytorch_model-00003-of-00006.bin",
"gpt_neox.layers.8.attention.query_key_value.weight": "pytorch_model-00003-of-00006.bin",
"gpt_neox.layers.8.attention.rotary_emb.inv_freq": "pytorch_model-00003-of-00006.bin",
"gpt_neox.layers.8.input_layernorm.bias": "pytorch_model-00003-of-00006.bin",
"gpt_neox.layers.8.input_layernorm.weight": "pytorch_model-00003-of-00006.bin",
"gpt_neox.layers.8.mlp.dense_4h_to_h.bias": "pytorch_model-00003-of-00006.bin",
"gpt_neox.layers.8.mlp.dense_4h_to_h.weight": "pytorch_model-00003-of-00006.bin",
"gpt_neox.layers.8.mlp.dense_h_to_4h.bias": "pytorch_model-00003-of-00006.bin",
"gpt_neox.layers.8.mlp.dense_h_to_4h.weight": "pytorch_model-00003-of-00006.bin",
"gpt_neox.layers.8.post_attention_layernorm.bias": "pytorch_model-00003-of-00006.bin",
"gpt_neox.layers.8.post_attention_layernorm.weight": "pytorch_model-00003-of-00006.bin",
"gpt_neox.layers.9.attention.bias": "pytorch_model-00003-of-00006.bin",
"gpt_neox.layers.9.attention.dense.bias": "pytorch_model-00003-of-00006.bin",
"gpt_neox.layers.9.attention.dense.weight": "pytorch_model-00003-of-00006.bin",
"gpt_neox.layers.9.attention.masked_bias": "pytorch_model-00003-of-00006.bin",
"gpt_neox.layers.9.attention.query_key_value.bias": "pytorch_model-00003-of-00006.bin",
"gpt_neox.layers.9.attention.query_key_value.weight": "pytorch_model-00003-of-00006.bin",
"gpt_neox.layers.9.attention.rotary_emb.inv_freq": "pytorch_model-00003-of-00006.bin",
"gpt_neox.layers.9.input_layernorm.bias": "pytorch_model-00003-of-00006.bin",
"gpt_neox.layers.9.input_layernorm.weight": "pytorch_model-00003-of-00006.bin",
"gpt_neox.layers.9.mlp.dense_4h_to_h.bias": "pytorch_model-00003-of-00006.bin",
"gpt_neox.layers.9.mlp.dense_4h_to_h.weight": "pytorch_model-00003-of-00006.bin",
"gpt_neox.layers.9.mlp.dense_h_to_4h.bias": "pytorch_model-00003-of-00006.bin",
"gpt_neox.layers.9.mlp.dense_h_to_4h.weight": "pytorch_model-00003-of-00006.bin",
"gpt_neox.layers.9.post_attention_layernorm.bias": "pytorch_model-00003-of-00006.bin",
"gpt_neox.layers.9.post_attention_layernorm.weight": "pytorch_model-00003-of-00006.bin"
}
}

5
special_tokens_map.json Normal file
View File

@@ -0,0 +1,5 @@
{
"bos_token": "<|endoftext|>",
"eos_token": "<|endoftext|>",
"unk_token": "<|endoftext|>"
}

100529
tokenizer.json Normal file
View File

File diff suppressed because it is too large Load Diff

9
tokenizer_config.json Normal file
View File

@@ -0,0 +1,9 @@
{
"add_prefix_space": false,
"bos_token": "<|endoftext|>",
"clean_up_tokenization_spaces": true,
"eos_token": "<|endoftext|>",
"model_max_length": 1000000000000000019884624838656,
"tokenizer_class": "GPTNeoXTokenizer",
"unk_token": "<|endoftext|>"
}