CarperAI/pythia-2.8b-deduped-4k
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

初始化项目，由ModelHub XC社区提供模型

Browse Source 
Model: CarperAI/pythia-2.8b-deduped-4k
Source: Original Platform
This commit is contained in:
ModelHub XC
2026-04-10 22:52:54 +08:00
commit ff10d56010
21 changed files with 756 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

61
.gitattributes vendored Normal file
View File

@@ -0,0 +1,61 @@
*.7z filter=lfs diff=lfs merge=lfs -text
*.arrow filter=lfs diff=lfs merge=lfs -text
*.bz2 filter=lfs diff=lfs merge=lfs -text
*.ftz filter=lfs diff=lfs merge=lfs -text
*.gz filter=lfs diff=lfs merge=lfs -text
*.h5 filter=lfs diff=lfs merge=lfs -text
*.joblib filter=lfs diff=lfs merge=lfs -text
*.lfs.* filter=lfs diff=lfs merge=lfs -text
*.msgpack filter=lfs diff=lfs merge=lfs -text
*.onnx filter=lfs diff=lfs merge=lfs -text
*.ot filter=lfs diff=lfs merge=lfs -text
*.parquet filter=lfs diff=lfs merge=lfs -text
*.pb filter=lfs diff=lfs merge=lfs -text
*.pt filter=lfs diff=lfs merge=lfs -text
*.pth filter=lfs diff=lfs merge=lfs -text
*.rar filter=lfs diff=lfs merge=lfs -text
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
*.tar.* filter=lfs diff=lfs merge=lfs -text
*.tflite filter=lfs diff=lfs merge=lfs -text
*.tgz filter=lfs diff=lfs merge=lfs -text
*.xz filter=lfs diff=lfs merge=lfs -text
*.zip filter=lfs diff=lfs merge=lfs -text
*.zstandard filter=lfs diff=lfs merge=lfs -text
*.tfevents* filter=lfs diff=lfs merge=lfs -text
*.db* filter=lfs diff=lfs merge=lfs -text
*.ark* filter=lfs diff=lfs merge=lfs -text
**/*ckpt*data* filter=lfs diff=lfs merge=lfs -text
**/*ckpt*.meta filter=lfs diff=lfs merge=lfs -text
**/*ckpt*.index filter=lfs diff=lfs merge=lfs -text
*.safetensors filter=lfs diff=lfs merge=lfs -text
*.ckpt filter=lfs diff=lfs merge=lfs -text
*.gguf* filter=lfs diff=lfs merge=lfs -text
*.ggml filter=lfs diff=lfs merge=lfs -text
*.llamafile* filter=lfs diff=lfs merge=lfs -text
*.pt2 filter=lfs diff=lfs merge=lfs -text
*.mlmodel filter=lfs diff=lfs merge=lfs -text
*.npy filter=lfs diff=lfs merge=lfs -text
*.npz filter=lfs diff=lfs merge=lfs -text
*.pickle filter=lfs diff=lfs merge=lfs -text
*.pkl filter=lfs diff=lfs merge=lfs -text
*.tar filter=lfs diff=lfs merge=lfs -text
*.wasm filter=lfs diff=lfs merge=lfs -text
*.zst filter=lfs diff=lfs merge=lfs -text
*tfevents* filter=lfs diff=lfs merge=lfs -text
pytorch_model-00005-of-00012.bin filter=lfs diff=lfs merge=lfs -text
pytorch_model-00007-of-00012.bin filter=lfs diff=lfs merge=lfs -text
pytorch_model-00012-of-00012.bin filter=lfs diff=lfs merge=lfs -text
pytorch_model-00010-of-00012.bin filter=lfs diff=lfs merge=lfs -text
tokenizer.json filter=lfs diff=lfs merge=lfs -text
pytorch_model-00009-of-00012.bin filter=lfs diff=lfs merge=lfs -text
pytorch_model-00008-of-00012.bin filter=lfs diff=lfs merge=lfs -text
pytorch_model-00003-of-00012.bin filter=lfs diff=lfs merge=lfs -text
pytorch_model-00006-of-00012.bin filter=lfs diff=lfs merge=lfs -text
pytorch_model-00001-of-00012.bin filter=lfs diff=lfs merge=lfs -text
pytorch_model-00002-of-00012.bin filter=lfs diff=lfs merge=lfs -text
pytorch_model-00011-of-00012.bin filter=lfs diff=lfs merge=lfs -text
pytorch_model-00004-of-00012.bin filter=lfs diff=lfs merge=lfs -text

119
README.md Normal file
View File

@@ -0,0 +1,119 @@
---
license: apache-2.0
datasets:
- EleutherAI/the_pile_deduplicated
language:
- en
---
Pythia-2.8B Deduped 4K is a [Pythia-2.8B Deduped](https://huggingface.co/EleutherAI/pythia-2.8b-deduped) model fine-tuned with a 4096 context length.
Training resumed from their 143,000 step checkpoint and continued on The Pile v1 Deduped (threshold=0.87).
This particular model is from a checkpoint captured at step 175,500 for an extra 134,217,728,000 tokens of training.
Note: Sequence length warmup was not used to move up from 2048 but, in hindsight, should have been applied.
## Config
```yaml
{
# 8 Nodes 8xA100 40GB
"eval_batch_size": 2,
"pipe-parallel-size": 1,
"model-parallel-size": 1,
"num-layers": 32,
"hidden-size": 2560,
"num-attention-heads": 32,
"seq-length": 4096,
"max-position-embeddings": 4096,
"norm": "layernorm",
"pos-emb": "rotary",
"rotary-pct": 0.25,
"no-weight-tying": true,
"gpt-j-residual": true,
"output-layer-parallelism": "column",
"init_method": "small_init",
"output_layer_init_method": "wang_init",
"attention-config": [[["flash"], 32]],
"scaled-upper-triang-masked-softmax-fusion": true,
"bias-gelu-fusion": true,
"optimizer": {
"type": "Adam",
"params": {
"lr": 1.6e-5,
"betas": [0.9, 0.95],
"eps": 1.0e-08
},
},
"min_lr": 8.0e-06,
"zero_optimization":{
"stage": 1,
"allgather_partitions": true,
"allgather_bucket_size": 500000000,
"overlap_comm": true,
"reduce_scatter": true,
"reduce_bucket_size": 500000000,
"contiguous_gradients": true,
"cpu_offload": false,
},
"train_micro_batch_size_per_gpu": 4,
"gradient-accumulation-steps": 4,
"data-impl": "mmap",
"checkpoint-activations": true,
"checkpoint-num-layers": 1,
"partition-activations": true,
"synchronize-each-layer": true,
"gradient_clipping": 1.0,
"weight-decay": 0.1,
"hidden-dropout": 0,
"attention-dropout": 0,
"fp16": {
"fp16": true,
"enabled": true,
"loss_scale": 0,
"loss_scale_window": 1000,
"initial_scale_power": 12,
"hysteresis": 2,
"min_loss_scale": 1,
},
"train-iters": 318000,
"lr-decay-iters": 318000,
"distributed-backend": "nccl",
"lr-decay-style": "cosine",
"warmup": 0.01,
"checkpoint-factor": 500,
"eval-interval": 50000,
"eval-iters": 10,
"extra-save-iters": [0, 512, 152001],
"train-data-paths": ["pile_0.87_deduped_text_document"],
"valid-data-paths": ["pile_0.87_deduped_text_document"],
"test-data-paths": ["pile_0.87_deduped_text_document"],
"tokenizer_type": "HFTokenizer",
"vocab-file": "20B_tokenizer.json",
"log-interval": 10,
"steps_per_print": 10,
"wall_clock_breakdown": true,
"log-grad-norm": true,
"launcher": "slurm",
"deepspeed_slurm": true,
}
```
## Acknoweldgements
This work would not have been possible without the support of [Stability AI](https://stability.ai/).

25
config.json Normal file
View File

@@ -0,0 +1,25 @@
{
"_name_or_path": "/fsx/ckpts/3b_pythia_tok=neox_data=pilev1-deduped-mixed/global_step175500_hf",
"architectures": [
"GPTNeoXForCausalLM"
],
"bos_token_id": 0,
"eos_token_id": 0,
"hidden_act": "gelu",
"hidden_size": 2560,
"initializer_range": 0.02,
"intermediate_size": 10240,
"layer_norm_eps": 1e-05,
"max_position_embeddings": 4096,
"model_type": "gpt_neox",
"num_attention_heads": 32,
"num_hidden_layers": 32,
"rotary_emb_base": 10000,
"rotary_pct": 0.25,
"tie_word_embeddings": false,
"torch_dtype": "float32",
"transformers_version": "4.27.4",
"use_cache": true,
"use_parallel_residual": true,
"vocab_size": 50304
}

1
configuration.json Normal file
View File

@@ -0,0 +1 @@
{"framework": "pytorch", "task": "text-generation", "allow_remote": true}

6
generation_config.json Normal file
View File

@@ -0,0 +1,6 @@
{
"_from_model_config": true,
"bos_token_id": 0,
"eos_token_id": 0,
"transformers_version": "4.27.4"
}

3
pytorch_model-00001-of-00012.bin Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:5523ea6a555782036e41d3dcd44bef083b89f145f701fd8550032dbd1235816f
size 968321909

3
pytorch_model-00002-of-00012.bin Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:9295685752d2acf61af300200f44ef826f9d3240eacf4eb8c5977bd70cbdeec4
size 994464505

3
pytorch_model-00003-of-00012.bin Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:1c806c4f7cfc1514ebe7850c56d4425d66b03a15af5a1f01ffec7345573b176f
size 994464505

3
pytorch_model-00004-of-00012.bin Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:9ac8eb4bf2d622936b50bbc54379971016e4c5b7e0c2d840399ca01fd8d5acd2
size 994464505

3
pytorch_model-00005-of-00012.bin Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:fdf117cc5a7a5a866da6c739a0ac9fde4c71313d252f593e28725372b4ace16a
size 994464505

3
pytorch_model-00006-of-00012.bin Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:738bb50239dfdcd8c79717619e4520a84a0438f00867a16316965948130b1386
size 994464505

3
pytorch_model-00007-of-00012.bin Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:883f3bb3257a3946787771b4fe4e9d55b3119e96753ed0270d203039867c08d0
size 994464505

3
pytorch_model-00008-of-00012.bin Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:1f57f7b50f363085b04998c115a4e2617ea7a6e47d4b08eac5d4e7ff36c62e6b
size 994464505

3
pytorch_model-00009-of-00012.bin Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:395a72fac28c420a207933369fb21062765b6d7a0b5b409d3f22a635acd10b86
size 994464505

3
pytorch_model-00010-of-00012.bin Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:7d4532a01efac576c23f9dd6950175139e93c10fcc903d40f1434bb558195fc7
size 994464505

3
pytorch_model-00011-of-00012.bin Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:61b805d07714506bff274f25b8b4422e5d6856aac8fea435f01be916e83d206d
size 994464505

3
pytorch_model-00012-of-00012.bin Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:b85a5757479e7b4c1c161bfeb8b10776ca4daac6c186587757e442f2cf67c2dc
size 724902608

491
pytorch_model.bin.index.json Normal file
View File

@@ -0,0 +1,491 @@
{
"metadata": {
"total_size": 11167946112.0
},
"weight_map": {
"embed_out.weight": "pytorch_model-00012-of-00012.bin",
"gpt_neox.embed_in.weight": "pytorch_model-00001-of-00012.bin",
"gpt_neox.final_layer_norm.bias": "pytorch_model-00012-of-00012.bin",
"gpt_neox.final_layer_norm.weight": "pytorch_model-00012-of-00012.bin",
"gpt_neox.layers.0.attention.bias": "pytorch_model-00001-of-00012.bin",
"gpt_neox.layers.0.attention.dense.bias": "pytorch_model-00001-of-00012.bin",
"gpt_neox.layers.0.attention.dense.weight": "pytorch_model-00001-of-00012.bin",
"gpt_neox.layers.0.attention.masked_bias": "pytorch_model-00001-of-00012.bin",
"gpt_neox.layers.0.attention.query_key_value.bias": "pytorch_model-00001-of-00012.bin",
"gpt_neox.layers.0.attention.query_key_value.weight": "pytorch_model-00001-of-00012.bin",
"gpt_neox.layers.0.attention.rotary_emb.inv_freq": "pytorch_model-00001-of-00012.bin",
"gpt_neox.layers.0.input_layernorm.bias": "pytorch_model-00001-of-00012.bin",
"gpt_neox.layers.0.input_layernorm.weight": "pytorch_model-00001-of-00012.bin",
"gpt_neox.layers.0.mlp.dense_4h_to_h.bias": "pytorch_model-00001-of-00012.bin",
"gpt_neox.layers.0.mlp.dense_4h_to_h.weight": "pytorch_model-00001-of-00012.bin",
"gpt_neox.layers.0.mlp.dense_h_to_4h.bias": "pytorch_model-00001-of-00012.bin",
"gpt_neox.layers.0.mlp.dense_h_to_4h.weight": "pytorch_model-00001-of-00012.bin",
"gpt_neox.layers.0.post_attention_layernorm.bias": "pytorch_model-00001-of-00012.bin",
"gpt_neox.layers.0.post_attention_layernorm.weight": "pytorch_model-00001-of-00012.bin",
"gpt_neox.layers.1.attention.bias": "pytorch_model-00001-of-00012.bin",
"gpt_neox.layers.1.attention.dense.bias": "pytorch_model-00001-of-00012.bin",
"gpt_neox.layers.1.attention.dense.weight": "pytorch_model-00001-of-00012.bin",
"gpt_neox.layers.1.attention.masked_bias": "pytorch_model-00001-of-00012.bin",
"gpt_neox.layers.1.attention.query_key_value.bias": "pytorch_model-00001-of-00012.bin",
"gpt_neox.layers.1.attention.query_key_value.weight": "pytorch_model-00001-of-00012.bin",
"gpt_neox.layers.1.attention.rotary_emb.inv_freq": "pytorch_model-00001-of-00012.bin",
"gpt_neox.layers.1.input_layernorm.bias": "pytorch_model-00001-of-00012.bin",
"gpt_neox.layers.1.input_layernorm.weight": "pytorch_model-00001-of-00012.bin",
"gpt_neox.layers.1.mlp.dense_4h_to_h.bias": "pytorch_model-00002-of-00012.bin",
"gpt_neox.layers.1.mlp.dense_4h_to_h.weight": "pytorch_model-00002-of-00012.bin",
"gpt_neox.layers.1.mlp.dense_h_to_4h.bias": "pytorch_model-00002-of-00012.bin",
"gpt_neox.layers.1.mlp.dense_h_to_4h.weight": "pytorch_model-00002-of-00012.bin",
"gpt_neox.layers.1.post_attention_layernorm.bias": "pytorch_model-00001-of-00012.bin",
"gpt_neox.layers.1.post_attention_layernorm.weight": "pytorch_model-00001-of-00012.bin",
"gpt_neox.layers.10.attention.bias": "pytorch_model-00004-of-00012.bin",
"gpt_neox.layers.10.attention.dense.bias": "pytorch_model-00004-of-00012.bin",
"gpt_neox.layers.10.attention.dense.weight": "pytorch_model-00004-of-00012.bin",
"gpt_neox.layers.10.attention.masked_bias": "pytorch_model-00004-of-00012.bin",
"gpt_neox.layers.10.attention.query_key_value.bias": "pytorch_model-00004-of-00012.bin",
"gpt_neox.layers.10.attention.query_key_value.weight": "pytorch_model-00004-of-00012.bin",
"gpt_neox.layers.10.attention.rotary_emb.inv_freq": "pytorch_model-00004-of-00012.bin",
"gpt_neox.layers.10.input_layernorm.bias": "pytorch_model-00004-of-00012.bin",
"gpt_neox.layers.10.input_layernorm.weight": "pytorch_model-00004-of-00012.bin",
"gpt_neox.layers.10.mlp.dense_4h_to_h.bias": "pytorch_model-00005-of-00012.bin",
"gpt_neox.layers.10.mlp.dense_4h_to_h.weight": "pytorch_model-00005-of-00012.bin",
"gpt_neox.layers.10.mlp.dense_h_to_4h.bias": "pytorch_model-00005-of-00012.bin",
"gpt_neox.layers.10.mlp.dense_h_to_4h.weight": "pytorch_model-00005-of-00012.bin",
"gpt_neox.layers.10.post_attention_layernorm.bias": "pytorch_model-00004-of-00012.bin",
"gpt_neox.layers.10.post_attention_layernorm.weight": "pytorch_model-00004-of-00012.bin",
"gpt_neox.layers.11.attention.bias": "pytorch_model-00005-of-00012.bin",
"gpt_neox.layers.11.attention.dense.bias": "pytorch_model-00005-of-00012.bin",
"gpt_neox.layers.11.attention.dense.weight": "pytorch_model-00005-of-00012.bin",
"gpt_neox.layers.11.attention.masked_bias": "pytorch_model-00005-of-00012.bin",
"gpt_neox.layers.11.attention.query_key_value.bias": "pytorch_model-00005-of-00012.bin",
"gpt_neox.layers.11.attention.query_key_value.weight": "pytorch_model-00005-of-00012.bin",
"gpt_neox.layers.11.attention.rotary_emb.inv_freq": "pytorch_model-00005-of-00012.bin",
"gpt_neox.layers.11.input_layernorm.bias": "pytorch_model-00005-of-00012.bin",
"gpt_neox.layers.11.input_layernorm.weight": "pytorch_model-00005-of-00012.bin",
"gpt_neox.layers.11.mlp.dense_4h_to_h.bias": "pytorch_model-00005-of-00012.bin",
"gpt_neox.layers.11.mlp.dense_4h_to_h.weight": "pytorch_model-00005-of-00012.bin",
"gpt_neox.layers.11.mlp.dense_h_to_4h.bias": "pytorch_model-00005-of-00012.bin",
"gpt_neox.layers.11.mlp.dense_h_to_4h.weight": "pytorch_model-00005-of-00012.bin",
"gpt_neox.layers.11.post_attention_layernorm.bias": "pytorch_model-00005-of-00012.bin",
"gpt_neox.layers.11.post_attention_layernorm.weight": "pytorch_model-00005-of-00012.bin",
"gpt_neox.layers.12.attention.bias": "pytorch_model-00005-of-00012.bin",
"gpt_neox.layers.12.attention.dense.bias": "pytorch_model-00005-of-00012.bin",
"gpt_neox.layers.12.attention.dense.weight": "pytorch_model-00005-of-00012.bin",
"gpt_neox.layers.12.attention.masked_bias": "pytorch_model-00005-of-00012.bin",
"gpt_neox.layers.12.attention.query_key_value.bias": "pytorch_model-00005-of-00012.bin",
"gpt_neox.layers.12.attention.query_key_value.weight": "pytorch_model-00005-of-00012.bin",
"gpt_neox.layers.12.attention.rotary_emb.inv_freq": "pytorch_model-00005-of-00012.bin",
"gpt_neox.layers.12.input_layernorm.bias": "pytorch_model-00005-of-00012.bin",
"gpt_neox.layers.12.input_layernorm.weight": "pytorch_model-00005-of-00012.bin",
"gpt_neox.layers.12.mlp.dense_4h_to_h.bias": "pytorch_model-00005-of-00012.bin",
"gpt_neox.layers.12.mlp.dense_4h_to_h.weight": "pytorch_model-00005-of-00012.bin",
"gpt_neox.layers.12.mlp.dense_h_to_4h.bias": "pytorch_model-00005-of-00012.bin",
"gpt_neox.layers.12.mlp.dense_h_to_4h.weight": "pytorch_model-00005-of-00012.bin",
"gpt_neox.layers.12.post_attention_layernorm.bias": "pytorch_model-00005-of-00012.bin",
"gpt_neox.layers.12.post_attention_layernorm.weight": "pytorch_model-00005-of-00012.bin",
"gpt_neox.layers.13.attention.bias": "pytorch_model-00005-of-00012.bin",
"gpt_neox.layers.13.attention.dense.bias": "pytorch_model-00005-of-00012.bin",
"gpt_neox.layers.13.attention.dense.weight": "pytorch_model-00005-of-00012.bin",
"gpt_neox.layers.13.attention.masked_bias": "pytorch_model-00005-of-00012.bin",
"gpt_neox.layers.13.attention.query_key_value.bias": "pytorch_model-00005-of-00012.bin",
"gpt_neox.layers.13.attention.query_key_value.weight": "pytorch_model-00005-of-00012.bin",
"gpt_neox.layers.13.attention.rotary_emb.inv_freq": "pytorch_model-00005-of-00012.bin",
"gpt_neox.layers.13.input_layernorm.bias": "pytorch_model-00005-of-00012.bin",
"gpt_neox.layers.13.input_layernorm.weight": "pytorch_model-00005-of-00012.bin",
"gpt_neox.layers.13.mlp.dense_4h_to_h.bias": "pytorch_model-00006-of-00012.bin",
"gpt_neox.layers.13.mlp.dense_4h_to_h.weight": "pytorch_model-00006-of-00012.bin",
"gpt_neox.layers.13.mlp.dense_h_to_4h.bias": "pytorch_model-00006-of-00012.bin",
"gpt_neox.layers.13.mlp.dense_h_to_4h.weight": "pytorch_model-00006-of-00012.bin",
"gpt_neox.layers.13.post_attention_layernorm.bias": "pytorch_model-00005-of-00012.bin",
"gpt_neox.layers.13.post_attention_layernorm.weight": "pytorch_model-00005-of-00012.bin",
"gpt_neox.layers.14.attention.bias": "pytorch_model-00006-of-00012.bin",
"gpt_neox.layers.14.attention.dense.bias": "pytorch_model-00006-of-00012.bin",
"gpt_neox.layers.14.attention.dense.weight": "pytorch_model-00006-of-00012.bin",
"gpt_neox.layers.14.attention.masked_bias": "pytorch_model-00006-of-00012.bin",
"gpt_neox.layers.14.attention.query_key_value.bias": "pytorch_model-00006-of-00012.bin",
"gpt_neox.layers.14.attention.query_key_value.weight": "pytorch_model-00006-of-00012.bin",
"gpt_neox.layers.14.attention.rotary_emb.inv_freq": "pytorch_model-00006-of-00012.bin",
"gpt_neox.layers.14.input_layernorm.bias": "pytorch_model-00006-of-00012.bin",
"gpt_neox.layers.14.input_layernorm.weight": "pytorch_model-00006-of-00012.bin",
"gpt_neox.layers.14.mlp.dense_4h_to_h.bias": "pytorch_model-00006-of-00012.bin",
"gpt_neox.layers.14.mlp.dense_4h_to_h.weight": "pytorch_model-00006-of-00012.bin",
"gpt_neox.layers.14.mlp.dense_h_to_4h.bias": "pytorch_model-00006-of-00012.bin",
"gpt_neox.layers.14.mlp.dense_h_to_4h.weight": "pytorch_model-00006-of-00012.bin",
"gpt_neox.layers.14.post_attention_layernorm.bias": "pytorch_model-00006-of-00012.bin",
"gpt_neox.layers.14.post_attention_layernorm.weight": "pytorch_model-00006-of-00012.bin",
"gpt_neox.layers.15.attention.bias": "pytorch_model-00006-of-00012.bin",
"gpt_neox.layers.15.attention.dense.bias": "pytorch_model-00006-of-00012.bin",
"gpt_neox.layers.15.attention.dense.weight": "pytorch_model-00006-of-00012.bin",
"gpt_neox.layers.15.attention.masked_bias": "pytorch_model-00006-of-00012.bin",
"gpt_neox.layers.15.attention.query_key_value.bias": "pytorch_model-00006-of-00012.bin",
"gpt_neox.layers.15.attention.query_key_value.weight": "pytorch_model-00006-of-00012.bin",
"gpt_neox.layers.15.attention.rotary_emb.inv_freq": "pytorch_model-00006-of-00012.bin",
"gpt_neox.layers.15.input_layernorm.bias": "pytorch_model-00006-of-00012.bin",
"gpt_neox.layers.15.input_layernorm.weight": "pytorch_model-00006-of-00012.bin",
"gpt_neox.layers.15.mlp.dense_4h_to_h.bias": "pytorch_model-00006-of-00012.bin",
"gpt_neox.layers.15.mlp.dense_4h_to_h.weight": "pytorch_model-00006-of-00012.bin",
"gpt_neox.layers.15.mlp.dense_h_to_4h.bias": "pytorch_model-00006-of-00012.bin",
"gpt_neox.layers.15.mlp.dense_h_to_4h.weight": "pytorch_model-00006-of-00012.bin",
"gpt_neox.layers.15.post_attention_layernorm.bias": "pytorch_model-00006-of-00012.bin",
"gpt_neox.layers.15.post_attention_layernorm.weight": "pytorch_model-00006-of-00012.bin",
"gpt_neox.layers.16.attention.bias": "pytorch_model-00006-of-00012.bin",
"gpt_neox.layers.16.attention.dense.bias": "pytorch_model-00006-of-00012.bin",
"gpt_neox.layers.16.attention.dense.weight": "pytorch_model-00006-of-00012.bin",
"gpt_neox.layers.16.attention.masked_bias": "pytorch_model-00006-of-00012.bin",
"gpt_neox.layers.16.attention.query_key_value.bias": "pytorch_model-00006-of-00012.bin",
"gpt_neox.layers.16.attention.query_key_value.weight": "pytorch_model-00006-of-00012.bin",
"gpt_neox.layers.16.attention.rotary_emb.inv_freq": "pytorch_model-00006-of-00012.bin",
"gpt_neox.layers.16.input_layernorm.bias": "pytorch_model-00006-of-00012.bin",
"gpt_neox.layers.16.input_layernorm.weight": "pytorch_model-00006-of-00012.bin",
"gpt_neox.layers.16.mlp.dense_4h_to_h.bias": "pytorch_model-00007-of-00012.bin",
"gpt_neox.layers.16.mlp.dense_4h_to_h.weight": "pytorch_model-00007-of-00012.bin",
"gpt_neox.layers.16.mlp.dense_h_to_4h.bias": "pytorch_model-00007-of-00012.bin",
"gpt_neox.layers.16.mlp.dense_h_to_4h.weight": "pytorch_model-00007-of-00012.bin",
"gpt_neox.layers.16.post_attention_layernorm.bias": "pytorch_model-00006-of-00012.bin",
"gpt_neox.layers.16.post_attention_layernorm.weight": "pytorch_model-00006-of-00012.bin",
"gpt_neox.layers.17.attention.bias": "pytorch_model-00007-of-00012.bin",
"gpt_neox.layers.17.attention.dense.bias": "pytorch_model-00007-of-00012.bin",
"gpt_neox.layers.17.attention.dense.weight": "pytorch_model-00007-of-00012.bin",
"gpt_neox.layers.17.attention.masked_bias": "pytorch_model-00007-of-00012.bin",
"gpt_neox.layers.17.attention.query_key_value.bias": "pytorch_model-00007-of-00012.bin",
"gpt_neox.layers.17.attention.query_key_value.weight": "pytorch_model-00007-of-00012.bin",
"gpt_neox.layers.17.attention.rotary_emb.inv_freq": "pytorch_model-00007-of-00012.bin",
"gpt_neox.layers.17.input_layernorm.bias": "pytorch_model-00007-of-00012.bin",
"gpt_neox.layers.17.input_layernorm.weight": "pytorch_model-00007-of-00012.bin",
"gpt_neox.layers.17.mlp.dense_4h_to_h.bias": "pytorch_model-00007-of-00012.bin",
"gpt_neox.layers.17.mlp.dense_4h_to_h.weight": "pytorch_model-00007-of-00012.bin",
"gpt_neox.layers.17.mlp.dense_h_to_4h.bias": "pytorch_model-00007-of-00012.bin",
"gpt_neox.layers.17.mlp.dense_h_to_4h.weight": "pytorch_model-00007-of-00012.bin",
"gpt_neox.layers.17.post_attention_layernorm.bias": "pytorch_model-00007-of-00012.bin",
"gpt_neox.layers.17.post_attention_layernorm.weight": "pytorch_model-00007-of-00012.bin",
"gpt_neox.layers.18.attention.bias": "pytorch_model-00007-of-00012.bin",
"gpt_neox.layers.18.attention.dense.bias": "pytorch_model-00007-of-00012.bin",
"gpt_neox.layers.18.attention.dense.weight": "pytorch_model-00007-of-00012.bin",
"gpt_neox.layers.18.attention.masked_bias": "pytorch_model-00007-of-00012.bin",
"gpt_neox.layers.18.attention.query_key_value.bias": "pytorch_model-00007-of-00012.bin",
"gpt_neox.layers.18.attention.query_key_value.weight": "pytorch_model-00007-of-00012.bin",
"gpt_neox.layers.18.attention.rotary_emb.inv_freq": "pytorch_model-00007-of-00012.bin",
"gpt_neox.layers.18.input_layernorm.bias": "pytorch_model-00007-of-00012.bin",
"gpt_neox.layers.18.input_layernorm.weight": "pytorch_model-00007-of-00012.bin",
"gpt_neox.layers.18.mlp.dense_4h_to_h.bias": "pytorch_model-00007-of-00012.bin",
"gpt_neox.layers.18.mlp.dense_4h_to_h.weight": "pytorch_model-00007-of-00012.bin",
"gpt_neox.layers.18.mlp.dense_h_to_4h.bias": "pytorch_model-00007-of-00012.bin",
"gpt_neox.layers.18.mlp.dense_h_to_4h.weight": "pytorch_model-00007-of-00012.bin",
"gpt_neox.layers.18.post_attention_layernorm.bias": "pytorch_model-00007-of-00012.bin",
"gpt_neox.layers.18.post_attention_layernorm.weight": "pytorch_model-00007-of-00012.bin",
"gpt_neox.layers.19.attention.bias": "pytorch_model-00007-of-00012.bin",
"gpt_neox.layers.19.attention.dense.bias": "pytorch_model-00007-of-00012.bin",
"gpt_neox.layers.19.attention.dense.weight": "pytorch_model-00007-of-00012.bin",
"gpt_neox.layers.19.attention.masked_bias": "pytorch_model-00007-of-00012.bin",
"gpt_neox.layers.19.attention.query_key_value.bias": "pytorch_model-00007-of-00012.bin",
"gpt_neox.layers.19.attention.query_key_value.weight": "pytorch_model-00007-of-00012.bin",
"gpt_neox.layers.19.attention.rotary_emb.inv_freq": "pytorch_model-00007-of-00012.bin",
"gpt_neox.layers.19.input_layernorm.bias": "pytorch_model-00007-of-00012.bin",
"gpt_neox.layers.19.input_layernorm.weight": "pytorch_model-00007-of-00012.bin",
"gpt_neox.layers.19.mlp.dense_4h_to_h.bias": "pytorch_model-00008-of-00012.bin",
"gpt_neox.layers.19.mlp.dense_4h_to_h.weight": "pytorch_model-00008-of-00012.bin",
"gpt_neox.layers.19.mlp.dense_h_to_4h.bias": "pytorch_model-00008-of-00012.bin",
"gpt_neox.layers.19.mlp.dense_h_to_4h.weight": "pytorch_model-00008-of-00012.bin",
"gpt_neox.layers.19.post_attention_layernorm.bias": "pytorch_model-00007-of-00012.bin",
"gpt_neox.layers.19.post_attention_layernorm.weight": "pytorch_model-00007-of-00012.bin",
"gpt_neox.layers.2.attention.bias": "pytorch_model-00002-of-00012.bin",
"gpt_neox.layers.2.attention.dense.bias": "pytorch_model-00002-of-00012.bin",
"gpt_neox.layers.2.attention.dense.weight": "pytorch_model-00002-of-00012.bin",
"gpt_neox.layers.2.attention.masked_bias": "pytorch_model-00002-of-00012.bin",
"gpt_neox.layers.2.attention.query_key_value.bias": "pytorch_model-00002-of-00012.bin",
"gpt_neox.layers.2.attention.query_key_value.weight": "pytorch_model-00002-of-00012.bin",
"gpt_neox.layers.2.attention.rotary_emb.inv_freq": "pytorch_model-00002-of-00012.bin",
"gpt_neox.layers.2.input_layernorm.bias": "pytorch_model-00002-of-00012.bin",
"gpt_neox.layers.2.input_layernorm.weight": "pytorch_model-00002-of-00012.bin",
"gpt_neox.layers.2.mlp.dense_4h_to_h.bias": "pytorch_model-00002-of-00012.bin",
"gpt_neox.layers.2.mlp.dense_4h_to_h.weight": "pytorch_model-00002-of-00012.bin",
"gpt_neox.layers.2.mlp.dense_h_to_4h.bias": "pytorch_model-00002-of-00012.bin",
"gpt_neox.layers.2.mlp.dense_h_to_4h.weight": "pytorch_model-00002-of-00012.bin",
"gpt_neox.layers.2.post_attention_layernorm.bias": "pytorch_model-00002-of-00012.bin",
"gpt_neox.layers.2.post_attention_layernorm.weight": "pytorch_model-00002-of-00012.bin",
"gpt_neox.layers.20.attention.bias": "pytorch_model-00008-of-00012.bin",
"gpt_neox.layers.20.attention.dense.bias": "pytorch_model-00008-of-00012.bin",
"gpt_neox.layers.20.attention.dense.weight": "pytorch_model-00008-of-00012.bin",
"gpt_neox.layers.20.attention.masked_bias": "pytorch_model-00008-of-00012.bin",
"gpt_neox.layers.20.attention.query_key_value.bias": "pytorch_model-00008-of-00012.bin",
"gpt_neox.layers.20.attention.query_key_value.weight": "pytorch_model-00008-of-00012.bin",
"gpt_neox.layers.20.attention.rotary_emb.inv_freq": "pytorch_model-00008-of-00012.bin",
"gpt_neox.layers.20.input_layernorm.bias": "pytorch_model-00008-of-00012.bin",
"gpt_neox.layers.20.input_layernorm.weight": "pytorch_model-00008-of-00012.bin",
"gpt_neox.layers.20.mlp.dense_4h_to_h.bias": "pytorch_model-00008-of-00012.bin",
"gpt_neox.layers.20.mlp.dense_4h_to_h.weight": "pytorch_model-00008-of-00012.bin",
"gpt_neox.layers.20.mlp.dense_h_to_4h.bias": "pytorch_model-00008-of-00012.bin",
"gpt_neox.layers.20.mlp.dense_h_to_4h.weight": "pytorch_model-00008-of-00012.bin",
"gpt_neox.layers.20.post_attention_layernorm.bias": "pytorch_model-00008-of-00012.bin",
"gpt_neox.layers.20.post_attention_layernorm.weight": "pytorch_model-00008-of-00012.bin",
"gpt_neox.layers.21.attention.bias": "pytorch_model-00008-of-00012.bin",
"gpt_neox.layers.21.attention.dense.bias": "pytorch_model-00008-of-00012.bin",
"gpt_neox.layers.21.attention.dense.weight": "pytorch_model-00008-of-00012.bin",
"gpt_neox.layers.21.attention.masked_bias": "pytorch_model-00008-of-00012.bin",
"gpt_neox.layers.21.attention.query_key_value.bias": "pytorch_model-00008-of-00012.bin",
"gpt_neox.layers.21.attention.query_key_value.weight": "pytorch_model-00008-of-00012.bin",
"gpt_neox.layers.21.attention.rotary_emb.inv_freq": "pytorch_model-00008-of-00012.bin",
"gpt_neox.layers.21.input_layernorm.bias": "pytorch_model-00008-of-00012.bin",
"gpt_neox.layers.21.input_layernorm.weight": "pytorch_model-00008-of-00012.bin",
"gpt_neox.layers.21.mlp.dense_4h_to_h.bias": "pytorch_model-00008-of-00012.bin",
"gpt_neox.layers.21.mlp.dense_4h_to_h.weight": "pytorch_model-00008-of-00012.bin",
"gpt_neox.layers.21.mlp.dense_h_to_4h.bias": "pytorch_model-00008-of-00012.bin",
"gpt_neox.layers.21.mlp.dense_h_to_4h.weight": "pytorch_model-00008-of-00012.bin",
"gpt_neox.layers.21.post_attention_layernorm.bias": "pytorch_model-00008-of-00012.bin",
"gpt_neox.layers.21.post_attention_layernorm.weight": "pytorch_model-00008-of-00012.bin",
"gpt_neox.layers.22.attention.bias": "pytorch_model-00008-of-00012.bin",
"gpt_neox.layers.22.attention.dense.bias": "pytorch_model-00008-of-00012.bin",
"gpt_neox.layers.22.attention.dense.weight": "pytorch_model-00008-of-00012.bin",
"gpt_neox.layers.22.attention.masked_bias": "pytorch_model-00008-of-00012.bin",
"gpt_neox.layers.22.attention.query_key_value.bias": "pytorch_model-00008-of-00012.bin",
"gpt_neox.layers.22.attention.query_key_value.weight": "pytorch_model-00008-of-00012.bin",
"gpt_neox.layers.22.attention.rotary_emb.inv_freq": "pytorch_model-00008-of-00012.bin",
"gpt_neox.layers.22.input_layernorm.bias": "pytorch_model-00008-of-00012.bin",
"gpt_neox.layers.22.input_layernorm.weight": "pytorch_model-00008-of-00012.bin",
"gpt_neox.layers.22.mlp.dense_4h_to_h.bias": "pytorch_model-00009-of-00012.bin",
"gpt_neox.layers.22.mlp.dense_4h_to_h.weight": "pytorch_model-00009-of-00012.bin",
"gpt_neox.layers.22.mlp.dense_h_to_4h.bias": "pytorch_model-00009-of-00012.bin",
"gpt_neox.layers.22.mlp.dense_h_to_4h.weight": "pytorch_model-00009-of-00012.bin",
"gpt_neox.layers.22.post_attention_layernorm.bias": "pytorch_model-00008-of-00012.bin",
"gpt_neox.layers.22.post_attention_layernorm.weight": "pytorch_model-00008-of-00012.bin",
"gpt_neox.layers.23.attention.bias": "pytorch_model-00009-of-00012.bin",
"gpt_neox.layers.23.attention.dense.bias": "pytorch_model-00009-of-00012.bin",
"gpt_neox.layers.23.attention.dense.weight": "pytorch_model-00009-of-00012.bin",
"gpt_neox.layers.23.attention.masked_bias": "pytorch_model-00009-of-00012.bin",
"gpt_neox.layers.23.attention.query_key_value.bias": "pytorch_model-00009-of-00012.bin",
"gpt_neox.layers.23.attention.query_key_value.weight": "pytorch_model-00009-of-00012.bin",
"gpt_neox.layers.23.attention.rotary_emb.inv_freq": "pytorch_model-00009-of-00012.bin",
"gpt_neox.layers.23.input_layernorm.bias": "pytorch_model-00009-of-00012.bin",
"gpt_neox.layers.23.input_layernorm.weight": "pytorch_model-00009-of-00012.bin",
"gpt_neox.layers.23.mlp.dense_4h_to_h.bias": "pytorch_model-00009-of-00012.bin",
"gpt_neox.layers.23.mlp.dense_4h_to_h.weight": "pytorch_model-00009-of-00012.bin",
"gpt_neox.layers.23.mlp.dense_h_to_4h.bias": "pytorch_model-00009-of-00012.bin",
"gpt_neox.layers.23.mlp.dense_h_to_4h.weight": "pytorch_model-00009-of-00012.bin",
"gpt_neox.layers.23.post_attention_layernorm.bias": "pytorch_model-00009-of-00012.bin",
"gpt_neox.layers.23.post_attention_layernorm.weight": "pytorch_model-00009-of-00012.bin",
"gpt_neox.layers.24.attention.bias": "pytorch_model-00009-of-00012.bin",
"gpt_neox.layers.24.attention.dense.bias": "pytorch_model-00009-of-00012.bin",
"gpt_neox.layers.24.attention.dense.weight": "pytorch_model-00009-of-00012.bin",
"gpt_neox.layers.24.attention.masked_bias": "pytorch_model-00009-of-00012.bin",
"gpt_neox.layers.24.attention.query_key_value.bias": "pytorch_model-00009-of-00012.bin",
"gpt_neox.layers.24.attention.query_key_value.weight": "pytorch_model-00009-of-00012.bin",
"gpt_neox.layers.24.attention.rotary_emb.inv_freq": "pytorch_model-00009-of-00012.bin",
"gpt_neox.layers.24.input_layernorm.bias": "pytorch_model-00009-of-00012.bin",
"gpt_neox.layers.24.input_layernorm.weight": "pytorch_model-00009-of-00012.bin",
"gpt_neox.layers.24.mlp.dense_4h_to_h.bias": "pytorch_model-00009-of-00012.bin",
"gpt_neox.layers.24.mlp.dense_4h_to_h.weight": "pytorch_model-00009-of-00012.bin",
"gpt_neox.layers.24.mlp.dense_h_to_4h.bias": "pytorch_model-00009-of-00012.bin",
"gpt_neox.layers.24.mlp.dense_h_to_4h.weight": "pytorch_model-00009-of-00012.bin",
"gpt_neox.layers.24.post_attention_layernorm.bias": "pytorch_model-00009-of-00012.bin",
"gpt_neox.layers.24.post_attention_layernorm.weight": "pytorch_model-00009-of-00012.bin",
"gpt_neox.layers.25.attention.bias": "pytorch_model-00009-of-00012.bin",
"gpt_neox.layers.25.attention.dense.bias": "pytorch_model-00009-of-00012.bin",
"gpt_neox.layers.25.attention.dense.weight": "pytorch_model-00009-of-00012.bin",
"gpt_neox.layers.25.attention.masked_bias": "pytorch_model-00009-of-00012.bin",
"gpt_neox.layers.25.attention.query_key_value.bias": "pytorch_model-00009-of-00012.bin",
"gpt_neox.layers.25.attention.query_key_value.weight": "pytorch_model-00009-of-00012.bin",
"gpt_neox.layers.25.attention.rotary_emb.inv_freq": "pytorch_model-00009-of-00012.bin",
"gpt_neox.layers.25.input_layernorm.bias": "pytorch_model-00009-of-00012.bin",
"gpt_neox.layers.25.input_layernorm.weight": "pytorch_model-00009-of-00012.bin",
"gpt_neox.layers.25.mlp.dense_4h_to_h.bias": "pytorch_model-00010-of-00012.bin",
"gpt_neox.layers.25.mlp.dense_4h_to_h.weight": "pytorch_model-00010-of-00012.bin",
"gpt_neox.layers.25.mlp.dense_h_to_4h.bias": "pytorch_model-00010-of-00012.bin",
"gpt_neox.layers.25.mlp.dense_h_to_4h.weight": "pytorch_model-00010-of-00012.bin",
"gpt_neox.layers.25.post_attention_layernorm.bias": "pytorch_model-00009-of-00012.bin",
"gpt_neox.layers.25.post_attention_layernorm.weight": "pytorch_model-00009-of-00012.bin",
"gpt_neox.layers.26.attention.bias": "pytorch_model-00010-of-00012.bin",
"gpt_neox.layers.26.attention.dense.bias": "pytorch_model-00010-of-00012.bin",
"gpt_neox.layers.26.attention.dense.weight": "pytorch_model-00010-of-00012.bin",
"gpt_neox.layers.26.attention.masked_bias": "pytorch_model-00010-of-00012.bin",
"gpt_neox.layers.26.attention.query_key_value.bias": "pytorch_model-00010-of-00012.bin",
"gpt_neox.layers.26.attention.query_key_value.weight": "pytorch_model-00010-of-00012.bin",
"gpt_neox.layers.26.attention.rotary_emb.inv_freq": "pytorch_model-00010-of-00012.bin",
"gpt_neox.layers.26.input_layernorm.bias": "pytorch_model-00010-of-00012.bin",
"gpt_neox.layers.26.input_layernorm.weight": "pytorch_model-00010-of-00012.bin",
"gpt_neox.layers.26.mlp.dense_4h_to_h.bias": "pytorch_model-00010-of-00012.bin",
"gpt_neox.layers.26.mlp.dense_4h_to_h.weight": "pytorch_model-00010-of-00012.bin",
"gpt_neox.layers.26.mlp.dense_h_to_4h.bias": "pytorch_model-00010-of-00012.bin",
"gpt_neox.layers.26.mlp.dense_h_to_4h.weight": "pytorch_model-00010-of-00012.bin",
"gpt_neox.layers.26.post_attention_layernorm.bias": "pytorch_model-00010-of-00012.bin",
"gpt_neox.layers.26.post_attention_layernorm.weight": "pytorch_model-00010-of-00012.bin",
"gpt_neox.layers.27.attention.bias": "pytorch_model-00010-of-00012.bin",
"gpt_neox.layers.27.attention.dense.bias": "pytorch_model-00010-of-00012.bin",
"gpt_neox.layers.27.attention.dense.weight": "pytorch_model-00010-of-00012.bin",
"gpt_neox.layers.27.attention.masked_bias": "pytorch_model-00010-of-00012.bin",
"gpt_neox.layers.27.attention.query_key_value.bias": "pytorch_model-00010-of-00012.bin",
"gpt_neox.layers.27.attention.query_key_value.weight": "pytorch_model-00010-of-00012.bin",
"gpt_neox.layers.27.attention.rotary_emb.inv_freq": "pytorch_model-00010-of-00012.bin",
"gpt_neox.layers.27.input_layernorm.bias": "pytorch_model-00010-of-00012.bin",
"gpt_neox.layers.27.input_layernorm.weight": "pytorch_model-00010-of-00012.bin",
"gpt_neox.layers.27.mlp.dense_4h_to_h.bias": "pytorch_model-00010-of-00012.bin",
"gpt_neox.layers.27.mlp.dense_4h_to_h.weight": "pytorch_model-00010-of-00012.bin",
"gpt_neox.layers.27.mlp.dense_h_to_4h.bias": "pytorch_model-00010-of-00012.bin",
"gpt_neox.layers.27.mlp.dense_h_to_4h.weight": "pytorch_model-00010-of-00012.bin",
"gpt_neox.layers.27.post_attention_layernorm.bias": "pytorch_model-00010-of-00012.bin",
"gpt_neox.layers.27.post_attention_layernorm.weight": "pytorch_model-00010-of-00012.bin",
"gpt_neox.layers.28.attention.bias": "pytorch_model-00010-of-00012.bin",
"gpt_neox.layers.28.attention.dense.bias": "pytorch_model-00010-of-00012.bin",
"gpt_neox.layers.28.attention.dense.weight": "pytorch_model-00010-of-00012.bin",
"gpt_neox.layers.28.attention.masked_bias": "pytorch_model-00010-of-00012.bin",
"gpt_neox.layers.28.attention.query_key_value.bias": "pytorch_model-00010-of-00012.bin",
"gpt_neox.layers.28.attention.query_key_value.weight": "pytorch_model-00010-of-00012.bin",
"gpt_neox.layers.28.attention.rotary_emb.inv_freq": "pytorch_model-00010-of-00012.bin",
"gpt_neox.layers.28.input_layernorm.bias": "pytorch_model-00010-of-00012.bin",
"gpt_neox.layers.28.input_layernorm.weight": "pytorch_model-00010-of-00012.bin",
"gpt_neox.layers.28.mlp.dense_4h_to_h.bias": "pytorch_model-00011-of-00012.bin",
"gpt_neox.layers.28.mlp.dense_4h_to_h.weight": "pytorch_model-00011-of-00012.bin",
"gpt_neox.layers.28.mlp.dense_h_to_4h.bias": "pytorch_model-00011-of-00012.bin",
"gpt_neox.layers.28.mlp.dense_h_to_4h.weight": "pytorch_model-00011-of-00012.bin",
"gpt_neox.layers.28.post_attention_layernorm.bias": "pytorch_model-00010-of-00012.bin",
"gpt_neox.layers.28.post_attention_layernorm.weight": "pytorch_model-00010-of-00012.bin",
"gpt_neox.layers.29.attention.bias": "pytorch_model-00011-of-00012.bin",
"gpt_neox.layers.29.attention.dense.bias": "pytorch_model-00011-of-00012.bin",
"gpt_neox.layers.29.attention.dense.weight": "pytorch_model-00011-of-00012.bin",
"gpt_neox.layers.29.attention.masked_bias": "pytorch_model-00011-of-00012.bin",
"gpt_neox.layers.29.attention.query_key_value.bias": "pytorch_model-00011-of-00012.bin",
"gpt_neox.layers.29.attention.query_key_value.weight": "pytorch_model-00011-of-00012.bin",
"gpt_neox.layers.29.attention.rotary_emb.inv_freq": "pytorch_model-00011-of-00012.bin",
"gpt_neox.layers.29.input_layernorm.bias": "pytorch_model-00011-of-00012.bin",
"gpt_neox.layers.29.input_layernorm.weight": "pytorch_model-00011-of-00012.bin",
"gpt_neox.layers.29.mlp.dense_4h_to_h.bias": "pytorch_model-00011-of-00012.bin",
"gpt_neox.layers.29.mlp.dense_4h_to_h.weight": "pytorch_model-00011-of-00012.bin",
"gpt_neox.layers.29.mlp.dense_h_to_4h.bias": "pytorch_model-00011-of-00012.bin",
"gpt_neox.layers.29.mlp.dense_h_to_4h.weight": "pytorch_model-00011-of-00012.bin",
"gpt_neox.layers.29.post_attention_layernorm.bias": "pytorch_model-00011-of-00012.bin",
"gpt_neox.layers.29.post_attention_layernorm.weight": "pytorch_model-00011-of-00012.bin",
"gpt_neox.layers.3.attention.bias": "pytorch_model-00002-of-00012.bin",
"gpt_neox.layers.3.attention.dense.bias": "pytorch_model-00002-of-00012.bin",
"gpt_neox.layers.3.attention.dense.weight": "pytorch_model-00002-of-00012.bin",
"gpt_neox.layers.3.attention.masked_bias": "pytorch_model-00002-of-00012.bin",
"gpt_neox.layers.3.attention.query_key_value.bias": "pytorch_model-00002-of-00012.bin",
"gpt_neox.layers.3.attention.query_key_value.weight": "pytorch_model-00002-of-00012.bin",
"gpt_neox.layers.3.attention.rotary_emb.inv_freq": "pytorch_model-00002-of-00012.bin",
"gpt_neox.layers.3.input_layernorm.bias": "pytorch_model-00002-of-00012.bin",
"gpt_neox.layers.3.input_layernorm.weight": "pytorch_model-00002-of-00012.bin",
"gpt_neox.layers.3.mlp.dense_4h_to_h.bias": "pytorch_model-00002-of-00012.bin",
"gpt_neox.layers.3.mlp.dense_4h_to_h.weight": "pytorch_model-00002-of-00012.bin",
"gpt_neox.layers.3.mlp.dense_h_to_4h.bias": "pytorch_model-00002-of-00012.bin",
"gpt_neox.layers.3.mlp.dense_h_to_4h.weight": "pytorch_model-00002-of-00012.bin",
"gpt_neox.layers.3.post_attention_layernorm.bias": "pytorch_model-00002-of-00012.bin",
"gpt_neox.layers.3.post_attention_layernorm.weight": "pytorch_model-00002-of-00012.bin",
"gpt_neox.layers.30.attention.bias": "pytorch_model-00011-of-00012.bin",
"gpt_neox.layers.30.attention.dense.bias": "pytorch_model-00011-of-00012.bin",
"gpt_neox.layers.30.attention.dense.weight": "pytorch_model-00011-of-00012.bin",
"gpt_neox.layers.30.attention.masked_bias": "pytorch_model-00011-of-00012.bin",
"gpt_neox.layers.30.attention.query_key_value.bias": "pytorch_model-00011-of-00012.bin",
"gpt_neox.layers.30.attention.query_key_value.weight": "pytorch_model-00011-of-00012.bin",
"gpt_neox.layers.30.attention.rotary_emb.inv_freq": "pytorch_model-00011-of-00012.bin",
"gpt_neox.layers.30.input_layernorm.bias": "pytorch_model-00011-of-00012.bin",
"gpt_neox.layers.30.input_layernorm.weight": "pytorch_model-00011-of-00012.bin",
"gpt_neox.layers.30.mlp.dense_4h_to_h.bias": "pytorch_model-00011-of-00012.bin",
"gpt_neox.layers.30.mlp.dense_4h_to_h.weight": "pytorch_model-00011-of-00012.bin",
"gpt_neox.layers.30.mlp.dense_h_to_4h.bias": "pytorch_model-00011-of-00012.bin",
"gpt_neox.layers.30.mlp.dense_h_to_4h.weight": "pytorch_model-00011-of-00012.bin",
"gpt_neox.layers.30.post_attention_layernorm.bias": "pytorch_model-00011-of-00012.bin",
"gpt_neox.layers.30.post_attention_layernorm.weight": "pytorch_model-00011-of-00012.bin",
"gpt_neox.layers.31.attention.bias": "pytorch_model-00011-of-00012.bin",
"gpt_neox.layers.31.attention.dense.bias": "pytorch_model-00011-of-00012.bin",
"gpt_neox.layers.31.attention.dense.weight": "pytorch_model-00011-of-00012.bin",
"gpt_neox.layers.31.attention.masked_bias": "pytorch_model-00011-of-00012.bin",
"gpt_neox.layers.31.attention.query_key_value.bias": "pytorch_model-00011-of-00012.bin",
"gpt_neox.layers.31.attention.query_key_value.weight": "pytorch_model-00011-of-00012.bin",
"gpt_neox.layers.31.attention.rotary_emb.inv_freq": "pytorch_model-00011-of-00012.bin",
"gpt_neox.layers.31.input_layernorm.bias": "pytorch_model-00011-of-00012.bin",
"gpt_neox.layers.31.input_layernorm.weight": "pytorch_model-00011-of-00012.bin",
"gpt_neox.layers.31.mlp.dense_4h_to_h.bias": "pytorch_model-00012-of-00012.bin",
"gpt_neox.layers.31.mlp.dense_4h_to_h.weight": "pytorch_model-00012-of-00012.bin",
"gpt_neox.layers.31.mlp.dense_h_to_4h.bias": "pytorch_model-00012-of-00012.bin",
"gpt_neox.layers.31.mlp.dense_h_to_4h.weight": "pytorch_model-00012-of-00012.bin",
"gpt_neox.layers.31.post_attention_layernorm.bias": "pytorch_model-00011-of-00012.bin",
"gpt_neox.layers.31.post_attention_layernorm.weight": "pytorch_model-00011-of-00012.bin",
"gpt_neox.layers.4.attention.bias": "pytorch_model-00002-of-00012.bin",
"gpt_neox.layers.4.attention.dense.bias": "pytorch_model-00002-of-00012.bin",
"gpt_neox.layers.4.attention.dense.weight": "pytorch_model-00002-of-00012.bin",
"gpt_neox.layers.4.attention.masked_bias": "pytorch_model-00002-of-00012.bin",
"gpt_neox.layers.4.attention.query_key_value.bias": "pytorch_model-00002-of-00012.bin",
"gpt_neox.layers.4.attention.query_key_value.weight": "pytorch_model-00002-of-00012.bin",
"gpt_neox.layers.4.attention.rotary_emb.inv_freq": "pytorch_model-00002-of-00012.bin",
"gpt_neox.layers.4.input_layernorm.bias": "pytorch_model-00002-of-00012.bin",
"gpt_neox.layers.4.input_layernorm.weight": "pytorch_model-00002-of-00012.bin",
"gpt_neox.layers.4.mlp.dense_4h_to_h.bias": "pytorch_model-00003-of-00012.bin",
"gpt_neox.layers.4.mlp.dense_4h_to_h.weight": "pytorch_model-00003-of-00012.bin",
"gpt_neox.layers.4.mlp.dense_h_to_4h.bias": "pytorch_model-00003-of-00012.bin",
"gpt_neox.layers.4.mlp.dense_h_to_4h.weight": "pytorch_model-00003-of-00012.bin",
"gpt_neox.layers.4.post_attention_layernorm.bias": "pytorch_model-00002-of-00012.bin",
"gpt_neox.layers.4.post_attention_layernorm.weight": "pytorch_model-00002-of-00012.bin",
"gpt_neox.layers.5.attention.bias": "pytorch_model-00003-of-00012.bin",
"gpt_neox.layers.5.attention.dense.bias": "pytorch_model-00003-of-00012.bin",
"gpt_neox.layers.5.attention.dense.weight": "pytorch_model-00003-of-00012.bin",
"gpt_neox.layers.5.attention.masked_bias": "pytorch_model-00003-of-00012.bin",
"gpt_neox.layers.5.attention.query_key_value.bias": "pytorch_model-00003-of-00012.bin",
"gpt_neox.layers.5.attention.query_key_value.weight": "pytorch_model-00003-of-00012.bin",
"gpt_neox.layers.5.attention.rotary_emb.inv_freq": "pytorch_model-00003-of-00012.bin",
"gpt_neox.layers.5.input_layernorm.bias": "pytorch_model-00003-of-00012.bin",
"gpt_neox.layers.5.input_layernorm.weight": "pytorch_model-00003-of-00012.bin",
"gpt_neox.layers.5.mlp.dense_4h_to_h.bias": "pytorch_model-00003-of-00012.bin",
"gpt_neox.layers.5.mlp.dense_4h_to_h.weight": "pytorch_model-00003-of-00012.bin",
"gpt_neox.layers.5.mlp.dense_h_to_4h.bias": "pytorch_model-00003-of-00012.bin",
"gpt_neox.layers.5.mlp.dense_h_to_4h.weight": "pytorch_model-00003-of-00012.bin",
"gpt_neox.layers.5.post_attention_layernorm.bias": "pytorch_model-00003-of-00012.bin",
"gpt_neox.layers.5.post_attention_layernorm.weight": "pytorch_model-00003-of-00012.bin",
"gpt_neox.layers.6.attention.bias": "pytorch_model-00003-of-00012.bin",
"gpt_neox.layers.6.attention.dense.bias": "pytorch_model-00003-of-00012.bin",
"gpt_neox.layers.6.attention.dense.weight": "pytorch_model-00003-of-00012.bin",
"gpt_neox.layers.6.attention.masked_bias": "pytorch_model-00003-of-00012.bin",
"gpt_neox.layers.6.attention.query_key_value.bias": "pytorch_model-00003-of-00012.bin",
"gpt_neox.layers.6.attention.query_key_value.weight": "pytorch_model-00003-of-00012.bin",
"gpt_neox.layers.6.attention.rotary_emb.inv_freq": "pytorch_model-00003-of-00012.bin",
"gpt_neox.layers.6.input_layernorm.bias": "pytorch_model-00003-of-00012.bin",
"gpt_neox.layers.6.input_layernorm.weight": "pytorch_model-00003-of-00012.bin",
"gpt_neox.layers.6.mlp.dense_4h_to_h.bias": "pytorch_model-00003-of-00012.bin",
"gpt_neox.layers.6.mlp.dense_4h_to_h.weight": "pytorch_model-00003-of-00012.bin",
"gpt_neox.layers.6.mlp.dense_h_to_4h.bias": "pytorch_model-00003-of-00012.bin",
"gpt_neox.layers.6.mlp.dense_h_to_4h.weight": "pytorch_model-00003-of-00012.bin",
"gpt_neox.layers.6.post_attention_layernorm.bias": "pytorch_model-00003-of-00012.bin",
"gpt_neox.layers.6.post_attention_layernorm.weight": "pytorch_model-00003-of-00012.bin",
"gpt_neox.layers.7.attention.bias": "pytorch_model-00003-of-00012.bin",
"gpt_neox.layers.7.attention.dense.bias": "pytorch_model-00003-of-00012.bin",
"gpt_neox.layers.7.attention.dense.weight": "pytorch_model-00003-of-00012.bin",
"gpt_neox.layers.7.attention.masked_bias": "pytorch_model-00003-of-00012.bin",
"gpt_neox.layers.7.attention.query_key_value.bias": "pytorch_model-00003-of-00012.bin",
"gpt_neox.layers.7.attention.query_key_value.weight": "pytorch_model-00003-of-00012.bin",
"gpt_neox.layers.7.attention.rotary_emb.inv_freq": "pytorch_model-00003-of-00012.bin",
"gpt_neox.layers.7.input_layernorm.bias": "pytorch_model-00003-of-00012.bin",
"gpt_neox.layers.7.input_layernorm.weight": "pytorch_model-00003-of-00012.bin",
"gpt_neox.layers.7.mlp.dense_4h_to_h.bias": "pytorch_model-00004-of-00012.bin",
"gpt_neox.layers.7.mlp.dense_4h_to_h.weight": "pytorch_model-00004-of-00012.bin",
"gpt_neox.layers.7.mlp.dense_h_to_4h.bias": "pytorch_model-00004-of-00012.bin",
"gpt_neox.layers.7.mlp.dense_h_to_4h.weight": "pytorch_model-00004-of-00012.bin",
"gpt_neox.layers.7.post_attention_layernorm.bias": "pytorch_model-00003-of-00012.bin",
"gpt_neox.layers.7.post_attention_layernorm.weight": "pytorch_model-00003-of-00012.bin",
"gpt_neox.layers.8.attention.bias": "pytorch_model-00004-of-00012.bin",
"gpt_neox.layers.8.attention.dense.bias": "pytorch_model-00004-of-00012.bin",
"gpt_neox.layers.8.attention.dense.weight": "pytorch_model-00004-of-00012.bin",
"gpt_neox.layers.8.attention.masked_bias": "pytorch_model-00004-of-00012.bin",
"gpt_neox.layers.8.attention.query_key_value.bias": "pytorch_model-00004-of-00012.bin",
"gpt_neox.layers.8.attention.query_key_value.weight": "pytorch_model-00004-of-00012.bin",
"gpt_neox.layers.8.attention.rotary_emb.inv_freq": "pytorch_model-00004-of-00012.bin",
"gpt_neox.layers.8.input_layernorm.bias": "pytorch_model-00004-of-00012.bin",
"gpt_neox.layers.8.input_layernorm.weight": "pytorch_model-00004-of-00012.bin",
"gpt_neox.layers.8.mlp.dense_4h_to_h.bias": "pytorch_model-00004-of-00012.bin",
"gpt_neox.layers.8.mlp.dense_4h_to_h.weight": "pytorch_model-00004-of-00012.bin",
"gpt_neox.layers.8.mlp.dense_h_to_4h.bias": "pytorch_model-00004-of-00012.bin",
"gpt_neox.layers.8.mlp.dense_h_to_4h.weight": "pytorch_model-00004-of-00012.bin",
"gpt_neox.layers.8.post_attention_layernorm.bias": "pytorch_model-00004-of-00012.bin",
"gpt_neox.layers.8.post_attention_layernorm.weight": "pytorch_model-00004-of-00012.bin",
"gpt_neox.layers.9.attention.bias": "pytorch_model-00004-of-00012.bin",
"gpt_neox.layers.9.attention.dense.bias": "pytorch_model-00004-of-00012.bin",
"gpt_neox.layers.9.attention.dense.weight": "pytorch_model-00004-of-00012.bin",
"gpt_neox.layers.9.attention.masked_bias": "pytorch_model-00004-of-00012.bin",
"gpt_neox.layers.9.attention.query_key_value.bias": "pytorch_model-00004-of-00012.bin",
"gpt_neox.layers.9.attention.query_key_value.weight": "pytorch_model-00004-of-00012.bin",
"gpt_neox.layers.9.attention.rotary_emb.inv_freq": "pytorch_model-00004-of-00012.bin",
"gpt_neox.layers.9.input_layernorm.bias": "pytorch_model-00004-of-00012.bin",
"gpt_neox.layers.9.input_layernorm.weight": "pytorch_model-00004-of-00012.bin",
"gpt_neox.layers.9.mlp.dense_4h_to_h.bias": "pytorch_model-00004-of-00012.bin",
"gpt_neox.layers.9.mlp.dense_4h_to_h.weight": "pytorch_model-00004-of-00012.bin",
"gpt_neox.layers.9.mlp.dense_h_to_4h.bias": "pytorch_model-00004-of-00012.bin",
"gpt_neox.layers.9.mlp.dense_h_to_4h.weight": "pytorch_model-00004-of-00012.bin",
"gpt_neox.layers.9.post_attention_layernorm.bias": "pytorch_model-00004-of-00012.bin",
"gpt_neox.layers.9.post_attention_layernorm.weight": "pytorch_model-00004-of-00012.bin"
}
}

5
special_tokens_map.json Normal file
View File

@@ -0,0 +1,5 @@
{
"bos_token": "<|endoftext|>",
"eos_token": "<|endoftext|>",
"unk_token": "<|endoftext|>"
}

3
tokenizer.json Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:c24618a1b3e6a38167beff1c72cffd126c3a66254347304b50547d12c5f25624
size 2113710

9
tokenizer_config.json Normal file
View File

@@ -0,0 +1,9 @@
{
"add_prefix_space": false,
"bos_token": "<|endoftext|>",
"eos_token": "<|endoftext|>",
"model_max_length": 1000000000000000019884624838656,
"special_tokens_map_file": "/fsx/home-guac/.cache/huggingface/transformers/models--EleutherAI--gpt-neox-20b/snapshots/4e49eadb5d14bd22f314ec3f45b69a87b88c7691/special_tokens_map.json",
"tokenizer_class": "GPTNeoXTokenizer",
"unk_token": "<|endoftext|>"
}