commit ff10d560105f3f381ba91337039e763b2c24e553
Author: ModelHub XC <noreply@modelhub.org.cn>
Date:   Fri Apr 10 22:52:54 2026 +0800

    初始化项目，由ModelHub XC社区提供模型
    
    Model: CarperAI/pythia-2.8b-deduped-4k
    Source: Original Platform

diff --git a/.gitattributes b/.gitattributes
new file mode 100644
index 0000000..54d8467
--- /dev/null
+++ b/.gitattributes
@@ -0,0 +1,61 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+ 
+ 
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+ 
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zstandard filter=lfs diff=lfs merge=lfs -text
+*.tfevents* filter=lfs diff=lfs merge=lfs -text
+*.db* filter=lfs diff=lfs merge=lfs -text
+*.ark* filter=lfs diff=lfs merge=lfs -text
+**/*ckpt*data* filter=lfs diff=lfs merge=lfs -text
+**/*ckpt*.meta filter=lfs diff=lfs merge=lfs -text
+**/*ckpt*.index filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.gguf* filter=lfs diff=lfs merge=lfs -text
+*.ggml filter=lfs diff=lfs merge=lfs -text
+*.llamafile* filter=lfs diff=lfs merge=lfs -text
+*.pt2 filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+
+pytorch_model-00005-of-00012.bin filter=lfs diff=lfs merge=lfs -text
+pytorch_model-00007-of-00012.bin filter=lfs diff=lfs merge=lfs -text
+pytorch_model-00012-of-00012.bin filter=lfs diff=lfs merge=lfs -text
+pytorch_model-00010-of-00012.bin filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text
+pytorch_model-00009-of-00012.bin filter=lfs diff=lfs merge=lfs -text
+pytorch_model-00008-of-00012.bin filter=lfs diff=lfs merge=lfs -text
+pytorch_model-00003-of-00012.bin filter=lfs diff=lfs merge=lfs -text
+pytorch_model-00006-of-00012.bin filter=lfs diff=lfs merge=lfs -text
+pytorch_model-00001-of-00012.bin filter=lfs diff=lfs merge=lfs -text
+pytorch_model-00002-of-00012.bin filter=lfs diff=lfs merge=lfs -text
+pytorch_model-00011-of-00012.bin filter=lfs diff=lfs merge=lfs -text
+pytorch_model-00004-of-00012.bin filter=lfs diff=lfs merge=lfs -text
\ No newline at end of file
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..7c515c8
--- /dev/null
+++ b/README.md
@@ -0,0 +1,119 @@
+---
+license: apache-2.0
+datasets:
+- EleutherAI/the_pile_deduplicated
+language:
+- en
+---
+
+Pythia-2.8B Deduped 4K is a [Pythia-2.8B Deduped](https://huggingface.co/EleutherAI/pythia-2.8b-deduped) model fine-tuned with a 4096 context length.
+Training resumed from their 143,000 step checkpoint and continued on The Pile v1 Deduped (threshold=0.87).
+This particular model is from a checkpoint captured at step 175,500 for an extra 134,217,728,000 tokens of training.
+
+Note: Sequence length warmup was not used to move up from 2048 but, in hindsight, should have been applied.
+
+
+## Config
+
+```yaml
+{
+  # 8 Nodes 8xA100 40GB
+  "eval_batch_size": 2,
+
+  "pipe-parallel-size": 1,
+  "model-parallel-size": 1,
+
+  "num-layers": 32,
+  "hidden-size": 2560,
+  "num-attention-heads": 32,
+  "seq-length": 4096,
+  "max-position-embeddings": 4096,
+
+  "norm": "layernorm",
+  "pos-emb": "rotary",
+  "rotary-pct": 0.25,
+  "no-weight-tying": true,
+  "gpt-j-residual": true,
+  "output-layer-parallelism": "column",
+
+  "init_method": "small_init",
+  "output_layer_init_method": "wang_init",
+  
+  "attention-config": [[["flash"], 32]],
+  "scaled-upper-triang-masked-softmax-fusion": true,
+  "bias-gelu-fusion": true,
+
+  "optimizer": {
+    "type": "Adam",
+    "params": {
+      "lr": 1.6e-5,
+      "betas": [0.9, 0.95],
+      "eps": 1.0e-08
+    },
+  },
+  "min_lr": 8.0e-06,
+
+  "zero_optimization":{
+      "stage": 1,
+      "allgather_partitions": true,
+      "allgather_bucket_size": 500000000,
+      "overlap_comm": true,
+      "reduce_scatter": true,
+      "reduce_bucket_size": 500000000,
+      "contiguous_gradients": true,
+      "cpu_offload": false,
+  },
+  "train_micro_batch_size_per_gpu": 4,
+  "gradient-accumulation-steps": 4,
+  "data-impl": "mmap",
+
+  "checkpoint-activations": true,
+  "checkpoint-num-layers": 1,
+  "partition-activations": true,
+  "synchronize-each-layer": true,
+
+  "gradient_clipping": 1.0,
+  "weight-decay": 0.1,
+  "hidden-dropout": 0,
+  "attention-dropout": 0,
+
+  "fp16": {
+      "fp16": true,
+      "enabled": true,
+      "loss_scale": 0,
+      "loss_scale_window": 1000,
+      "initial_scale_power": 12,
+      "hysteresis": 2,
+      "min_loss_scale": 1,
+  },
+
+  "train-iters": 318000, 
+  "lr-decay-iters": 318000,
+  "distributed-backend": "nccl",
+  "lr-decay-style": "cosine",
+  "warmup": 0.01,
+  "checkpoint-factor": 500,
+  "eval-interval": 50000,
+  "eval-iters": 10,
+  "extra-save-iters": [0, 512, 152001],
+
+  "train-data-paths": ["pile_0.87_deduped_text_document"],
+  "valid-data-paths": ["pile_0.87_deduped_text_document"],
+  "test-data-paths": ["pile_0.87_deduped_text_document"],
+
+  "tokenizer_type": "HFTokenizer",
+  "vocab-file": "20B_tokenizer.json",
+
+  "log-interval": 10,
+  "steps_per_print": 10,
+  "wall_clock_breakdown": true,
+  "log-grad-norm": true,
+
+  "launcher": "slurm",
+  "deepspeed_slurm": true,
+}
+```
+
+## Acknoweldgements
+
+This work would not have been possible without the support of [Stability AI](https://stability.ai/).
\ No newline at end of file
diff --git a/config.json b/config.json
new file mode 100644
index 0000000..53ce428
--- /dev/null
+++ b/config.json
@@ -0,0 +1,25 @@
+{
+  "_name_or_path": "/fsx/ckpts/3b_pythia_tok=neox_data=pilev1-deduped-mixed/global_step175500_hf",
+  "architectures": [
+    "GPTNeoXForCausalLM"
+  ],
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "hidden_act": "gelu",
+  "hidden_size": 2560,
+  "initializer_range": 0.02,
+  "intermediate_size": 10240,
+  "layer_norm_eps": 1e-05,
+  "max_position_embeddings": 4096,
+  "model_type": "gpt_neox",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "rotary_emb_base": 10000,
+  "rotary_pct": 0.25,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float32",
+  "transformers_version": "4.27.4",
+  "use_cache": true,
+  "use_parallel_residual": true,
+  "vocab_size": 50304
+}
diff --git a/configuration.json b/configuration.json
new file mode 100644
index 0000000..bbeeda1
--- /dev/null
+++ b/configuration.json
@@ -0,0 +1 @@
+{"framework": "pytorch", "task": "text-generation", "allow_remote": true}
\ No newline at end of file
diff --git a/generation_config.json b/generation_config.json
new file mode 100644
index 0000000..9c8cb71
--- /dev/null
+++ b/generation_config.json
@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "transformers_version": "4.27.4"
+}
diff --git a/pytorch_model-00001-of-00012.bin b/pytorch_model-00001-of-00012.bin
new file mode 100644
index 0000000..a4570d0
--- /dev/null
+++ b/pytorch_model-00001-of-00012.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5523ea6a555782036e41d3dcd44bef083b89f145f701fd8550032dbd1235816f
+size 968321909
diff --git a/pytorch_model-00002-of-00012.bin b/pytorch_model-00002-of-00012.bin
new file mode 100644
index 0000000..25a07db
--- /dev/null
+++ b/pytorch_model-00002-of-00012.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9295685752d2acf61af300200f44ef826f9d3240eacf4eb8c5977bd70cbdeec4
+size 994464505
diff --git a/pytorch_model-00003-of-00012.bin b/pytorch_model-00003-of-00012.bin
new file mode 100644
index 0000000..8fb1dc1
--- /dev/null
+++ b/pytorch_model-00003-of-00012.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1c806c4f7cfc1514ebe7850c56d4425d66b03a15af5a1f01ffec7345573b176f
+size 994464505
diff --git a/pytorch_model-00004-of-00012.bin b/pytorch_model-00004-of-00012.bin
new file mode 100644
index 0000000..3d84d56
--- /dev/null
+++ b/pytorch_model-00004-of-00012.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9ac8eb4bf2d622936b50bbc54379971016e4c5b7e0c2d840399ca01fd8d5acd2
+size 994464505
diff --git a/pytorch_model-00005-of-00012.bin b/pytorch_model-00005-of-00012.bin
new file mode 100644
index 0000000..d2ba14c
--- /dev/null
+++ b/pytorch_model-00005-of-00012.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fdf117cc5a7a5a866da6c739a0ac9fde4c71313d252f593e28725372b4ace16a
+size 994464505
diff --git a/pytorch_model-00006-of-00012.bin b/pytorch_model-00006-of-00012.bin
new file mode 100644
index 0000000..2b3e1b0
--- /dev/null
+++ b/pytorch_model-00006-of-00012.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:738bb50239dfdcd8c79717619e4520a84a0438f00867a16316965948130b1386
+size 994464505
diff --git a/pytorch_model-00007-of-00012.bin b/pytorch_model-00007-of-00012.bin
new file mode 100644
index 0000000..431d3cc
--- /dev/null
+++ b/pytorch_model-00007-of-00012.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:883f3bb3257a3946787771b4fe4e9d55b3119e96753ed0270d203039867c08d0
+size 994464505
diff --git a/pytorch_model-00008-of-00012.bin b/pytorch_model-00008-of-00012.bin
new file mode 100644
index 0000000..d32e7d9
--- /dev/null
+++ b/pytorch_model-00008-of-00012.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1f57f7b50f363085b04998c115a4e2617ea7a6e47d4b08eac5d4e7ff36c62e6b
+size 994464505
diff --git a/pytorch_model-00009-of-00012.bin b/pytorch_model-00009-of-00012.bin
new file mode 100644
index 0000000..7c5ab45
--- /dev/null
+++ b/pytorch_model-00009-of-00012.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:395a72fac28c420a207933369fb21062765b6d7a0b5b409d3f22a635acd10b86
+size 994464505
diff --git a/pytorch_model-00010-of-00012.bin b/pytorch_model-00010-of-00012.bin
new file mode 100644
index 0000000..5974bac
--- /dev/null
+++ b/pytorch_model-00010-of-00012.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7d4532a01efac576c23f9dd6950175139e93c10fcc903d40f1434bb558195fc7
+size 994464505
diff --git a/pytorch_model-00011-of-00012.bin b/pytorch_model-00011-of-00012.bin
new file mode 100644
index 0000000..b94011b
--- /dev/null
+++ b/pytorch_model-00011-of-00012.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:61b805d07714506bff274f25b8b4422e5d6856aac8fea435f01be916e83d206d
+size 994464505
diff --git a/pytorch_model-00012-of-00012.bin b/pytorch_model-00012-of-00012.bin
new file mode 100644
index 0000000..bb314c2
--- /dev/null
+++ b/pytorch_model-00012-of-00012.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b85a5757479e7b4c1c161bfeb8b10776ca4daac6c186587757e442f2cf67c2dc
+size 724902608
diff --git a/pytorch_model.bin.index.json b/pytorch_model.bin.index.json
new file mode 100644
index 0000000..a7e249d
--- /dev/null
+++ b/pytorch_model.bin.index.json
@@ -0,0 +1,491 @@
+{
+  "metadata": {
+    "total_size": 11167946112.0
+  },
+  "weight_map": {
+    "embed_out.weight": "pytorch_model-00012-of-00012.bin",
+    "gpt_neox.embed_in.weight": "pytorch_model-00001-of-00012.bin",
+    "gpt_neox.final_layer_norm.bias": "pytorch_model-00012-of-00012.bin",
+    "gpt_neox.final_layer_norm.weight": "pytorch_model-00012-of-00012.bin",
+    "gpt_neox.layers.0.attention.bias": "pytorch_model-00001-of-00012.bin",
+    "gpt_neox.layers.0.attention.dense.bias": "pytorch_model-00001-of-00012.bin",
+    "gpt_neox.layers.0.attention.dense.weight": "pytorch_model-00001-of-00012.bin",
+    "gpt_neox.layers.0.attention.masked_bias": "pytorch_model-00001-of-00012.bin",
+    "gpt_neox.layers.0.attention.query_key_value.bias": "pytorch_model-00001-of-00012.bin",
+    "gpt_neox.layers.0.attention.query_key_value.weight": "pytorch_model-00001-of-00012.bin",
+    "gpt_neox.layers.0.attention.rotary_emb.inv_freq": "pytorch_model-00001-of-00012.bin",
+    "gpt_neox.layers.0.input_layernorm.bias": "pytorch_model-00001-of-00012.bin",
+    "gpt_neox.layers.0.input_layernorm.weight": "pytorch_model-00001-of-00012.bin",
+    "gpt_neox.layers.0.mlp.dense_4h_to_h.bias": "pytorch_model-00001-of-00012.bin",
+    "gpt_neox.layers.0.mlp.dense_4h_to_h.weight": "pytorch_model-00001-of-00012.bin",
+    "gpt_neox.layers.0.mlp.dense_h_to_4h.bias": "pytorch_model-00001-of-00012.bin",
+    "gpt_neox.layers.0.mlp.dense_h_to_4h.weight": "pytorch_model-00001-of-00012.bin",
+    "gpt_neox.layers.0.post_attention_layernorm.bias": "pytorch_model-00001-of-00012.bin",
+    "gpt_neox.layers.0.post_attention_layernorm.weight": "pytorch_model-00001-of-00012.bin",
+    "gpt_neox.layers.1.attention.bias": "pytorch_model-00001-of-00012.bin",
+    "gpt_neox.layers.1.attention.dense.bias": "pytorch_model-00001-of-00012.bin",
+    "gpt_neox.layers.1.attention.dense.weight": "pytorch_model-00001-of-00012.bin",
+    "gpt_neox.layers.1.attention.masked_bias": "pytorch_model-00001-of-00012.bin",
+    "gpt_neox.layers.1.attention.query_key_value.bias": "pytorch_model-00001-of-00012.bin",
+    "gpt_neox.layers.1.attention.query_key_value.weight": "pytorch_model-00001-of-00012.bin",
+    "gpt_neox.layers.1.attention.rotary_emb.inv_freq": "pytorch_model-00001-of-00012.bin",
+    "gpt_neox.layers.1.input_layernorm.bias": "pytorch_model-00001-of-00012.bin",
+    "gpt_neox.layers.1.input_layernorm.weight": "pytorch_model-00001-of-00012.bin",
+    "gpt_neox.layers.1.mlp.dense_4h_to_h.bias": "pytorch_model-00002-of-00012.bin",
+    "gpt_neox.layers.1.mlp.dense_4h_to_h.weight": "pytorch_model-00002-of-00012.bin",
+    "gpt_neox.layers.1.mlp.dense_h_to_4h.bias": "pytorch_model-00002-of-00012.bin",
+    "gpt_neox.layers.1.mlp.dense_h_to_4h.weight": "pytorch_model-00002-of-00012.bin",
+    "gpt_neox.layers.1.post_attention_layernorm.bias": "pytorch_model-00001-of-00012.bin",
+    "gpt_neox.layers.1.post_attention_layernorm.weight": "pytorch_model-00001-of-00012.bin",
+    "gpt_neox.layers.10.attention.bias": "pytorch_model-00004-of-00012.bin",
+    "gpt_neox.layers.10.attention.dense.bias": "pytorch_model-00004-of-00012.bin",
+    "gpt_neox.layers.10.attention.dense.weight": "pytorch_model-00004-of-00012.bin",
+    "gpt_neox.layers.10.attention.masked_bias": "pytorch_model-00004-of-00012.bin",
+    "gpt_neox.layers.10.attention.query_key_value.bias": "pytorch_model-00004-of-00012.bin",
+    "gpt_neox.layers.10.attention.query_key_value.weight": "pytorch_model-00004-of-00012.bin",
+    "gpt_neox.layers.10.attention.rotary_emb.inv_freq": "pytorch_model-00004-of-00012.bin",
+    "gpt_neox.layers.10.input_layernorm.bias": "pytorch_model-00004-of-00012.bin",
+    "gpt_neox.layers.10.input_layernorm.weight": "pytorch_model-00004-of-00012.bin",
+    "gpt_neox.layers.10.mlp.dense_4h_to_h.bias": "pytorch_model-00005-of-00012.bin",
+    "gpt_neox.layers.10.mlp.dense_4h_to_h.weight": "pytorch_model-00005-of-00012.bin",
+    "gpt_neox.layers.10.mlp.dense_h_to_4h.bias": "pytorch_model-00005-of-00012.bin",
+    "gpt_neox.layers.10.mlp.dense_h_to_4h.weight": "pytorch_model-00005-of-00012.bin",
+    "gpt_neox.layers.10.post_attention_layernorm.bias": "pytorch_model-00004-of-00012.bin",
+    "gpt_neox.layers.10.post_attention_layernorm.weight": "pytorch_model-00004-of-00012.bin",
+    "gpt_neox.layers.11.attention.bias": "pytorch_model-00005-of-00012.bin",
+    "gpt_neox.layers.11.attention.dense.bias": "pytorch_model-00005-of-00012.bin",
+    "gpt_neox.layers.11.attention.dense.weight": "pytorch_model-00005-of-00012.bin",
+    "gpt_neox.layers.11.attention.masked_bias": "pytorch_model-00005-of-00012.bin",
+    "gpt_neox.layers.11.attention.query_key_value.bias": "pytorch_model-00005-of-00012.bin",
+    "gpt_neox.layers.11.attention.query_key_value.weight": "pytorch_model-00005-of-00012.bin",
+    "gpt_neox.layers.11.attention.rotary_emb.inv_freq": "pytorch_model-00005-of-00012.bin",
+    "gpt_neox.layers.11.input_layernorm.bias": "pytorch_model-00005-of-00012.bin",
+    "gpt_neox.layers.11.input_layernorm.weight": "pytorch_model-00005-of-00012.bin",
+    "gpt_neox.layers.11.mlp.dense_4h_to_h.bias": "pytorch_model-00005-of-00012.bin",
+    "gpt_neox.layers.11.mlp.dense_4h_to_h.weight": "pytorch_model-00005-of-00012.bin",
+    "gpt_neox.layers.11.mlp.dense_h_to_4h.bias": "pytorch_model-00005-of-00012.bin",
+    "gpt_neox.layers.11.mlp.dense_h_to_4h.weight": "pytorch_model-00005-of-00012.bin",
+    "gpt_neox.layers.11.post_attention_layernorm.bias": "pytorch_model-00005-of-00012.bin",
+    "gpt_neox.layers.11.post_attention_layernorm.weight": "pytorch_model-00005-of-00012.bin",
+    "gpt_neox.layers.12.attention.bias": "pytorch_model-00005-of-00012.bin",
+    "gpt_neox.layers.12.attention.dense.bias": "pytorch_model-00005-of-00012.bin",
+    "gpt_neox.layers.12.attention.dense.weight": "pytorch_model-00005-of-00012.bin",
+    "gpt_neox.layers.12.attention.masked_bias": "pytorch_model-00005-of-00012.bin",
+    "gpt_neox.layers.12.attention.query_key_value.bias": "pytorch_model-00005-of-00012.bin",
+    "gpt_neox.layers.12.attention.query_key_value.weight": "pytorch_model-00005-of-00012.bin",
+    "gpt_neox.layers.12.attention.rotary_emb.inv_freq": "pytorch_model-00005-of-00012.bin",
+    "gpt_neox.layers.12.input_layernorm.bias": "pytorch_model-00005-of-00012.bin",
+    "gpt_neox.layers.12.input_layernorm.weight": "pytorch_model-00005-of-00012.bin",
+    "gpt_neox.layers.12.mlp.dense_4h_to_h.bias": "pytorch_model-00005-of-00012.bin",
+    "gpt_neox.layers.12.mlp.dense_4h_to_h.weight": "pytorch_model-00005-of-00012.bin",
+    "gpt_neox.layers.12.mlp.dense_h_to_4h.bias": "pytorch_model-00005-of-00012.bin",
+    "gpt_neox.layers.12.mlp.dense_h_to_4h.weight": "pytorch_model-00005-of-00012.bin",
+    "gpt_neox.layers.12.post_attention_layernorm.bias": "pytorch_model-00005-of-00012.bin",
+    "gpt_neox.layers.12.post_attention_layernorm.weight": "pytorch_model-00005-of-00012.bin",
+    "gpt_neox.layers.13.attention.bias": "pytorch_model-00005-of-00012.bin",
+    "gpt_neox.layers.13.attention.dense.bias": "pytorch_model-00005-of-00012.bin",
+    "gpt_neox.layers.13.attention.dense.weight": "pytorch_model-00005-of-00012.bin",
+    "gpt_neox.layers.13.attention.masked_bias": "pytorch_model-00005-of-00012.bin",
+    "gpt_neox.layers.13.attention.query_key_value.bias": "pytorch_model-00005-of-00012.bin",
+    "gpt_neox.layers.13.attention.query_key_value.weight": "pytorch_model-00005-of-00012.bin",
+    "gpt_neox.layers.13.attention.rotary_emb.inv_freq": "pytorch_model-00005-of-00012.bin",
+    "gpt_neox.layers.13.input_layernorm.bias": "pytorch_model-00005-of-00012.bin",
+    "gpt_neox.layers.13.input_layernorm.weight": "pytorch_model-00005-of-00012.bin",
+    "gpt_neox.layers.13.mlp.dense_4h_to_h.bias": "pytorch_model-00006-of-00012.bin",
+    "gpt_neox.layers.13.mlp.dense_4h_to_h.weight": "pytorch_model-00006-of-00012.bin",
+    "gpt_neox.layers.13.mlp.dense_h_to_4h.bias": "pytorch_model-00006-of-00012.bin",
+    "gpt_neox.layers.13.mlp.dense_h_to_4h.weight": "pytorch_model-00006-of-00012.bin",
+    "gpt_neox.layers.13.post_attention_layernorm.bias": "pytorch_model-00005-of-00012.bin",
+    "gpt_neox.layers.13.post_attention_layernorm.weight": "pytorch_model-00005-of-00012.bin",
+    "gpt_neox.layers.14.attention.bias": "pytorch_model-00006-of-00012.bin",
+    "gpt_neox.layers.14.attention.dense.bias": "pytorch_model-00006-of-00012.bin",
+    "gpt_neox.layers.14.attention.dense.weight": "pytorch_model-00006-of-00012.bin",
+    "gpt_neox.layers.14.attention.masked_bias": "pytorch_model-00006-of-00012.bin",
+    "gpt_neox.layers.14.attention.query_key_value.bias": "pytorch_model-00006-of-00012.bin",
+    "gpt_neox.layers.14.attention.query_key_value.weight": "pytorch_model-00006-of-00012.bin",
+    "gpt_neox.layers.14.attention.rotary_emb.inv_freq": "pytorch_model-00006-of-00012.bin",
+    "gpt_neox.layers.14.input_layernorm.bias": "pytorch_model-00006-of-00012.bin",
+    "gpt_neox.layers.14.input_layernorm.weight": "pytorch_model-00006-of-00012.bin",
+    "gpt_neox.layers.14.mlp.dense_4h_to_h.bias": "pytorch_model-00006-of-00012.bin",
+    "gpt_neox.layers.14.mlp.dense_4h_to_h.weight": "pytorch_model-00006-of-00012.bin",
+    "gpt_neox.layers.14.mlp.dense_h_to_4h.bias": "pytorch_model-00006-of-00012.bin",
+    "gpt_neox.layers.14.mlp.dense_h_to_4h.weight": "pytorch_model-00006-of-00012.bin",
+    "gpt_neox.layers.14.post_attention_layernorm.bias": "pytorch_model-00006-of-00012.bin",
+    "gpt_neox.layers.14.post_attention_layernorm.weight": "pytorch_model-00006-of-00012.bin",
+    "gpt_neox.layers.15.attention.bias": "pytorch_model-00006-of-00012.bin",
+    "gpt_neox.layers.15.attention.dense.bias": "pytorch_model-00006-of-00012.bin",
+    "gpt_neox.layers.15.attention.dense.weight": "pytorch_model-00006-of-00012.bin",
+    "gpt_neox.layers.15.attention.masked_bias": "pytorch_model-00006-of-00012.bin",
+    "gpt_neox.layers.15.attention.query_key_value.bias": "pytorch_model-00006-of-00012.bin",
+    "gpt_neox.layers.15.attention.query_key_value.weight": "pytorch_model-00006-of-00012.bin",
+    "gpt_neox.layers.15.attention.rotary_emb.inv_freq": "pytorch_model-00006-of-00012.bin",
+    "gpt_neox.layers.15.input_layernorm.bias": "pytorch_model-00006-of-00012.bin",
+    "gpt_neox.layers.15.input_layernorm.weight": "pytorch_model-00006-of-00012.bin",
+    "gpt_neox.layers.15.mlp.dense_4h_to_h.bias": "pytorch_model-00006-of-00012.bin",
+    "gpt_neox.layers.15.mlp.dense_4h_to_h.weight": "pytorch_model-00006-of-00012.bin",
+    "gpt_neox.layers.15.mlp.dense_h_to_4h.bias": "pytorch_model-00006-of-00012.bin",
+    "gpt_neox.layers.15.mlp.dense_h_to_4h.weight": "pytorch_model-00006-of-00012.bin",
+    "gpt_neox.layers.15.post_attention_layernorm.bias": "pytorch_model-00006-of-00012.bin",
+    "gpt_neox.layers.15.post_attention_layernorm.weight": "pytorch_model-00006-of-00012.bin",
+    "gpt_neox.layers.16.attention.bias": "pytorch_model-00006-of-00012.bin",
+    "gpt_neox.layers.16.attention.dense.bias": "pytorch_model-00006-of-00012.bin",
+    "gpt_neox.layers.16.attention.dense.weight": "pytorch_model-00006-of-00012.bin",
+    "gpt_neox.layers.16.attention.masked_bias": "pytorch_model-00006-of-00012.bin",
+    "gpt_neox.layers.16.attention.query_key_value.bias": "pytorch_model-00006-of-00012.bin",
+    "gpt_neox.layers.16.attention.query_key_value.weight": "pytorch_model-00006-of-00012.bin",
+    "gpt_neox.layers.16.attention.rotary_emb.inv_freq": "pytorch_model-00006-of-00012.bin",
+    "gpt_neox.layers.16.input_layernorm.bias": "pytorch_model-00006-of-00012.bin",
+    "gpt_neox.layers.16.input_layernorm.weight": "pytorch_model-00006-of-00012.bin",
+    "gpt_neox.layers.16.mlp.dense_4h_to_h.bias": "pytorch_model-00007-of-00012.bin",
+    "gpt_neox.layers.16.mlp.dense_4h_to_h.weight": "pytorch_model-00007-of-00012.bin",
+    "gpt_neox.layers.16.mlp.dense_h_to_4h.bias": "pytorch_model-00007-of-00012.bin",
+    "gpt_neox.layers.16.mlp.dense_h_to_4h.weight": "pytorch_model-00007-of-00012.bin",
+    "gpt_neox.layers.16.post_attention_layernorm.bias": "pytorch_model-00006-of-00012.bin",
+    "gpt_neox.layers.16.post_attention_layernorm.weight": "pytorch_model-00006-of-00012.bin",
+    "gpt_neox.layers.17.attention.bias": "pytorch_model-00007-of-00012.bin",
+    "gpt_neox.layers.17.attention.dense.bias": "pytorch_model-00007-of-00012.bin",
+    "gpt_neox.layers.17.attention.dense.weight": "pytorch_model-00007-of-00012.bin",
+    "gpt_neox.layers.17.attention.masked_bias": "pytorch_model-00007-of-00012.bin",
+    "gpt_neox.layers.17.attention.query_key_value.bias": "pytorch_model-00007-of-00012.bin",
+    "gpt_neox.layers.17.attention.query_key_value.weight": "pytorch_model-00007-of-00012.bin",
+    "gpt_neox.layers.17.attention.rotary_emb.inv_freq": "pytorch_model-00007-of-00012.bin",
+    "gpt_neox.layers.17.input_layernorm.bias": "pytorch_model-00007-of-00012.bin",
+    "gpt_neox.layers.17.input_layernorm.weight": "pytorch_model-00007-of-00012.bin",
+    "gpt_neox.layers.17.mlp.dense_4h_to_h.bias": "pytorch_model-00007-of-00012.bin",
+    "gpt_neox.layers.17.mlp.dense_4h_to_h.weight": "pytorch_model-00007-of-00012.bin",
+    "gpt_neox.layers.17.mlp.dense_h_to_4h.bias": "pytorch_model-00007-of-00012.bin",
+    "gpt_neox.layers.17.mlp.dense_h_to_4h.weight": "pytorch_model-00007-of-00012.bin",
+    "gpt_neox.layers.17.post_attention_layernorm.bias": "pytorch_model-00007-of-00012.bin",
+    "gpt_neox.layers.17.post_attention_layernorm.weight": "pytorch_model-00007-of-00012.bin",
+    "gpt_neox.layers.18.attention.bias": "pytorch_model-00007-of-00012.bin",
+    "gpt_neox.layers.18.attention.dense.bias": "pytorch_model-00007-of-00012.bin",
+    "gpt_neox.layers.18.attention.dense.weight": "pytorch_model-00007-of-00012.bin",
+    "gpt_neox.layers.18.attention.masked_bias": "pytorch_model-00007-of-00012.bin",
+    "gpt_neox.layers.18.attention.query_key_value.bias": "pytorch_model-00007-of-00012.bin",
+    "gpt_neox.layers.18.attention.query_key_value.weight": "pytorch_model-00007-of-00012.bin",
+    "gpt_neox.layers.18.attention.rotary_emb.inv_freq": "pytorch_model-00007-of-00012.bin",
+    "gpt_neox.layers.18.input_layernorm.bias": "pytorch_model-00007-of-00012.bin",
+    "gpt_neox.layers.18.input_layernorm.weight": "pytorch_model-00007-of-00012.bin",
+    "gpt_neox.layers.18.mlp.dense_4h_to_h.bias": "pytorch_model-00007-of-00012.bin",
+    "gpt_neox.layers.18.mlp.dense_4h_to_h.weight": "pytorch_model-00007-of-00012.bin",
+    "gpt_neox.layers.18.mlp.dense_h_to_4h.bias": "pytorch_model-00007-of-00012.bin",
+    "gpt_neox.layers.18.mlp.dense_h_to_4h.weight": "pytorch_model-00007-of-00012.bin",
+    "gpt_neox.layers.18.post_attention_layernorm.bias": "pytorch_model-00007-of-00012.bin",
+    "gpt_neox.layers.18.post_attention_layernorm.weight": "pytorch_model-00007-of-00012.bin",
+    "gpt_neox.layers.19.attention.bias": "pytorch_model-00007-of-00012.bin",
+    "gpt_neox.layers.19.attention.dense.bias": "pytorch_model-00007-of-00012.bin",
+    "gpt_neox.layers.19.attention.dense.weight": "pytorch_model-00007-of-00012.bin",
+    "gpt_neox.layers.19.attention.masked_bias": "pytorch_model-00007-of-00012.bin",
+    "gpt_neox.layers.19.attention.query_key_value.bias": "pytorch_model-00007-of-00012.bin",
+    "gpt_neox.layers.19.attention.query_key_value.weight": "pytorch_model-00007-of-00012.bin",
+    "gpt_neox.layers.19.attention.rotary_emb.inv_freq": "pytorch_model-00007-of-00012.bin",
+    "gpt_neox.layers.19.input_layernorm.bias": "pytorch_model-00007-of-00012.bin",
+    "gpt_neox.layers.19.input_layernorm.weight": "pytorch_model-00007-of-00012.bin",
+    "gpt_neox.layers.19.mlp.dense_4h_to_h.bias": "pytorch_model-00008-of-00012.bin",
+    "gpt_neox.layers.19.mlp.dense_4h_to_h.weight": "pytorch_model-00008-of-00012.bin",
+    "gpt_neox.layers.19.mlp.dense_h_to_4h.bias": "pytorch_model-00008-of-00012.bin",
+    "gpt_neox.layers.19.mlp.dense_h_to_4h.weight": "pytorch_model-00008-of-00012.bin",
+    "gpt_neox.layers.19.post_attention_layernorm.bias": "pytorch_model-00007-of-00012.bin",
+    "gpt_neox.layers.19.post_attention_layernorm.weight": "pytorch_model-00007-of-00012.bin",
+    "gpt_neox.layers.2.attention.bias": "pytorch_model-00002-of-00012.bin",
+    "gpt_neox.layers.2.attention.dense.bias": "pytorch_model-00002-of-00012.bin",
+    "gpt_neox.layers.2.attention.dense.weight": "pytorch_model-00002-of-00012.bin",
+    "gpt_neox.layers.2.attention.masked_bias": "pytorch_model-00002-of-00012.bin",
+    "gpt_neox.layers.2.attention.query_key_value.bias": "pytorch_model-00002-of-00012.bin",
+    "gpt_neox.layers.2.attention.query_key_value.weight": "pytorch_model-00002-of-00012.bin",
+    "gpt_neox.layers.2.attention.rotary_emb.inv_freq": "pytorch_model-00002-of-00012.bin",
+    "gpt_neox.layers.2.input_layernorm.bias": "pytorch_model-00002-of-00012.bin",
+    "gpt_neox.layers.2.input_layernorm.weight": "pytorch_model-00002-of-00012.bin",
+    "gpt_neox.layers.2.mlp.dense_4h_to_h.bias": "pytorch_model-00002-of-00012.bin",
+    "gpt_neox.layers.2.mlp.dense_4h_to_h.weight": "pytorch_model-00002-of-00012.bin",
+    "gpt_neox.layers.2.mlp.dense_h_to_4h.bias": "pytorch_model-00002-of-00012.bin",
+    "gpt_neox.layers.2.mlp.dense_h_to_4h.weight": "pytorch_model-00002-of-00012.bin",
+    "gpt_neox.layers.2.post_attention_layernorm.bias": "pytorch_model-00002-of-00012.bin",
+    "gpt_neox.layers.2.post_attention_layernorm.weight": "pytorch_model-00002-of-00012.bin",
+    "gpt_neox.layers.20.attention.bias": "pytorch_model-00008-of-00012.bin",
+    "gpt_neox.layers.20.attention.dense.bias": "pytorch_model-00008-of-00012.bin",
+    "gpt_neox.layers.20.attention.dense.weight": "pytorch_model-00008-of-00012.bin",
+    "gpt_neox.layers.20.attention.masked_bias": "pytorch_model-00008-of-00012.bin",
+    "gpt_neox.layers.20.attention.query_key_value.bias": "pytorch_model-00008-of-00012.bin",
+    "gpt_neox.layers.20.attention.query_key_value.weight": "pytorch_model-00008-of-00012.bin",
+    "gpt_neox.layers.20.attention.rotary_emb.inv_freq": "pytorch_model-00008-of-00012.bin",
+    "gpt_neox.layers.20.input_layernorm.bias": "pytorch_model-00008-of-00012.bin",
+    "gpt_neox.layers.20.input_layernorm.weight": "pytorch_model-00008-of-00012.bin",
+    "gpt_neox.layers.20.mlp.dense_4h_to_h.bias": "pytorch_model-00008-of-00012.bin",
+    "gpt_neox.layers.20.mlp.dense_4h_to_h.weight": "pytorch_model-00008-of-00012.bin",
+    "gpt_neox.layers.20.mlp.dense_h_to_4h.bias": "pytorch_model-00008-of-00012.bin",
+    "gpt_neox.layers.20.mlp.dense_h_to_4h.weight": "pytorch_model-00008-of-00012.bin",
+    "gpt_neox.layers.20.post_attention_layernorm.bias": "pytorch_model-00008-of-00012.bin",
+    "gpt_neox.layers.20.post_attention_layernorm.weight": "pytorch_model-00008-of-00012.bin",
+    "gpt_neox.layers.21.attention.bias": "pytorch_model-00008-of-00012.bin",
+    "gpt_neox.layers.21.attention.dense.bias": "pytorch_model-00008-of-00012.bin",
+    "gpt_neox.layers.21.attention.dense.weight": "pytorch_model-00008-of-00012.bin",
+    "gpt_neox.layers.21.attention.masked_bias": "pytorch_model-00008-of-00012.bin",
+    "gpt_neox.layers.21.attention.query_key_value.bias": "pytorch_model-00008-of-00012.bin",
+    "gpt_neox.layers.21.attention.query_key_value.weight": "pytorch_model-00008-of-00012.bin",
+    "gpt_neox.layers.21.attention.rotary_emb.inv_freq": "pytorch_model-00008-of-00012.bin",
+    "gpt_neox.layers.21.input_layernorm.bias": "pytorch_model-00008-of-00012.bin",
+    "gpt_neox.layers.21.input_layernorm.weight": "pytorch_model-00008-of-00012.bin",
+    "gpt_neox.layers.21.mlp.dense_4h_to_h.bias": "pytorch_model-00008-of-00012.bin",
+    "gpt_neox.layers.21.mlp.dense_4h_to_h.weight": "pytorch_model-00008-of-00012.bin",
+    "gpt_neox.layers.21.mlp.dense_h_to_4h.bias": "pytorch_model-00008-of-00012.bin",
+    "gpt_neox.layers.21.mlp.dense_h_to_4h.weight": "pytorch_model-00008-of-00012.bin",
+    "gpt_neox.layers.21.post_attention_layernorm.bias": "pytorch_model-00008-of-00012.bin",
+    "gpt_neox.layers.21.post_attention_layernorm.weight": "pytorch_model-00008-of-00012.bin",
+    "gpt_neox.layers.22.attention.bias": "pytorch_model-00008-of-00012.bin",
+    "gpt_neox.layers.22.attention.dense.bias": "pytorch_model-00008-of-00012.bin",
+    "gpt_neox.layers.22.attention.dense.weight": "pytorch_model-00008-of-00012.bin",
+    "gpt_neox.layers.22.attention.masked_bias": "pytorch_model-00008-of-00012.bin",
+    "gpt_neox.layers.22.attention.query_key_value.bias": "pytorch_model-00008-of-00012.bin",
+    "gpt_neox.layers.22.attention.query_key_value.weight": "pytorch_model-00008-of-00012.bin",
+    "gpt_neox.layers.22.attention.rotary_emb.inv_freq": "pytorch_model-00008-of-00012.bin",
+    "gpt_neox.layers.22.input_layernorm.bias": "pytorch_model-00008-of-00012.bin",
+    "gpt_neox.layers.22.input_layernorm.weight": "pytorch_model-00008-of-00012.bin",
+    "gpt_neox.layers.22.mlp.dense_4h_to_h.bias": "pytorch_model-00009-of-00012.bin",
+    "gpt_neox.layers.22.mlp.dense_4h_to_h.weight": "pytorch_model-00009-of-00012.bin",
+    "gpt_neox.layers.22.mlp.dense_h_to_4h.bias": "pytorch_model-00009-of-00012.bin",
+    "gpt_neox.layers.22.mlp.dense_h_to_4h.weight": "pytorch_model-00009-of-00012.bin",
+    "gpt_neox.layers.22.post_attention_layernorm.bias": "pytorch_model-00008-of-00012.bin",
+    "gpt_neox.layers.22.post_attention_layernorm.weight": "pytorch_model-00008-of-00012.bin",
+    "gpt_neox.layers.23.attention.bias": "pytorch_model-00009-of-00012.bin",
+    "gpt_neox.layers.23.attention.dense.bias": "pytorch_model-00009-of-00012.bin",
+    "gpt_neox.layers.23.attention.dense.weight": "pytorch_model-00009-of-00012.bin",
+    "gpt_neox.layers.23.attention.masked_bias": "pytorch_model-00009-of-00012.bin",
+    "gpt_neox.layers.23.attention.query_key_value.bias": "pytorch_model-00009-of-00012.bin",
+    "gpt_neox.layers.23.attention.query_key_value.weight": "pytorch_model-00009-of-00012.bin",
+    "gpt_neox.layers.23.attention.rotary_emb.inv_freq": "pytorch_model-00009-of-00012.bin",
+    "gpt_neox.layers.23.input_layernorm.bias": "pytorch_model-00009-of-00012.bin",
+    "gpt_neox.layers.23.input_layernorm.weight": "pytorch_model-00009-of-00012.bin",
+    "gpt_neox.layers.23.mlp.dense_4h_to_h.bias": "pytorch_model-00009-of-00012.bin",
+    "gpt_neox.layers.23.mlp.dense_4h_to_h.weight": "pytorch_model-00009-of-00012.bin",
+    "gpt_neox.layers.23.mlp.dense_h_to_4h.bias": "pytorch_model-00009-of-00012.bin",
+    "gpt_neox.layers.23.mlp.dense_h_to_4h.weight": "pytorch_model-00009-of-00012.bin",
+    "gpt_neox.layers.23.post_attention_layernorm.bias": "pytorch_model-00009-of-00012.bin",
+    "gpt_neox.layers.23.post_attention_layernorm.weight": "pytorch_model-00009-of-00012.bin",
+    "gpt_neox.layers.24.attention.bias": "pytorch_model-00009-of-00012.bin",
+    "gpt_neox.layers.24.attention.dense.bias": "pytorch_model-00009-of-00012.bin",
+    "gpt_neox.layers.24.attention.dense.weight": "pytorch_model-00009-of-00012.bin",
+    "gpt_neox.layers.24.attention.masked_bias": "pytorch_model-00009-of-00012.bin",
+    "gpt_neox.layers.24.attention.query_key_value.bias": "pytorch_model-00009-of-00012.bin",
+    "gpt_neox.layers.24.attention.query_key_value.weight": "pytorch_model-00009-of-00012.bin",
+    "gpt_neox.layers.24.attention.rotary_emb.inv_freq": "pytorch_model-00009-of-00012.bin",
+    "gpt_neox.layers.24.input_layernorm.bias": "pytorch_model-00009-of-00012.bin",
+    "gpt_neox.layers.24.input_layernorm.weight": "pytorch_model-00009-of-00012.bin",
+    "gpt_neox.layers.24.mlp.dense_4h_to_h.bias": "pytorch_model-00009-of-00012.bin",
+    "gpt_neox.layers.24.mlp.dense_4h_to_h.weight": "pytorch_model-00009-of-00012.bin",
+    "gpt_neox.layers.24.mlp.dense_h_to_4h.bias": "pytorch_model-00009-of-00012.bin",
+    "gpt_neox.layers.24.mlp.dense_h_to_4h.weight": "pytorch_model-00009-of-00012.bin",
+    "gpt_neox.layers.24.post_attention_layernorm.bias": "pytorch_model-00009-of-00012.bin",
+    "gpt_neox.layers.24.post_attention_layernorm.weight": "pytorch_model-00009-of-00012.bin",
+    "gpt_neox.layers.25.attention.bias": "pytorch_model-00009-of-00012.bin",
+    "gpt_neox.layers.25.attention.dense.bias": "pytorch_model-00009-of-00012.bin",
+    "gpt_neox.layers.25.attention.dense.weight": "pytorch_model-00009-of-00012.bin",
+    "gpt_neox.layers.25.attention.masked_bias": "pytorch_model-00009-of-00012.bin",
+    "gpt_neox.layers.25.attention.query_key_value.bias": "pytorch_model-00009-of-00012.bin",
+    "gpt_neox.layers.25.attention.query_key_value.weight": "pytorch_model-00009-of-00012.bin",
+    "gpt_neox.layers.25.attention.rotary_emb.inv_freq": "pytorch_model-00009-of-00012.bin",
+    "gpt_neox.layers.25.input_layernorm.bias": "pytorch_model-00009-of-00012.bin",
+    "gpt_neox.layers.25.input_layernorm.weight": "pytorch_model-00009-of-00012.bin",
+    "gpt_neox.layers.25.mlp.dense_4h_to_h.bias": "pytorch_model-00010-of-00012.bin",
+    "gpt_neox.layers.25.mlp.dense_4h_to_h.weight": "pytorch_model-00010-of-00012.bin",
+    "gpt_neox.layers.25.mlp.dense_h_to_4h.bias": "pytorch_model-00010-of-00012.bin",
+    "gpt_neox.layers.25.mlp.dense_h_to_4h.weight": "pytorch_model-00010-of-00012.bin",
+    "gpt_neox.layers.25.post_attention_layernorm.bias": "pytorch_model-00009-of-00012.bin",
+    "gpt_neox.layers.25.post_attention_layernorm.weight": "pytorch_model-00009-of-00012.bin",
+    "gpt_neox.layers.26.attention.bias": "pytorch_model-00010-of-00012.bin",
+    "gpt_neox.layers.26.attention.dense.bias": "pytorch_model-00010-of-00012.bin",
+    "gpt_neox.layers.26.attention.dense.weight": "pytorch_model-00010-of-00012.bin",
+    "gpt_neox.layers.26.attention.masked_bias": "pytorch_model-00010-of-00012.bin",
+    "gpt_neox.layers.26.attention.query_key_value.bias": "pytorch_model-00010-of-00012.bin",
+    "gpt_neox.layers.26.attention.query_key_value.weight": "pytorch_model-00010-of-00012.bin",
+    "gpt_neox.layers.26.attention.rotary_emb.inv_freq": "pytorch_model-00010-of-00012.bin",
+    "gpt_neox.layers.26.input_layernorm.bias": "pytorch_model-00010-of-00012.bin",
+    "gpt_neox.layers.26.input_layernorm.weight": "pytorch_model-00010-of-00012.bin",
+    "gpt_neox.layers.26.mlp.dense_4h_to_h.bias": "pytorch_model-00010-of-00012.bin",
+    "gpt_neox.layers.26.mlp.dense_4h_to_h.weight": "pytorch_model-00010-of-00012.bin",
+    "gpt_neox.layers.26.mlp.dense_h_to_4h.bias": "pytorch_model-00010-of-00012.bin",
+    "gpt_neox.layers.26.mlp.dense_h_to_4h.weight": "pytorch_model-00010-of-00012.bin",
+    "gpt_neox.layers.26.post_attention_layernorm.bias": "pytorch_model-00010-of-00012.bin",
+    "gpt_neox.layers.26.post_attention_layernorm.weight": "pytorch_model-00010-of-00012.bin",
+    "gpt_neox.layers.27.attention.bias": "pytorch_model-00010-of-00012.bin",
+    "gpt_neox.layers.27.attention.dense.bias": "pytorch_model-00010-of-00012.bin",
+    "gpt_neox.layers.27.attention.dense.weight": "pytorch_model-00010-of-00012.bin",
+    "gpt_neox.layers.27.attention.masked_bias": "pytorch_model-00010-of-00012.bin",
+    "gpt_neox.layers.27.attention.query_key_value.bias": "pytorch_model-00010-of-00012.bin",
+    "gpt_neox.layers.27.attention.query_key_value.weight": "pytorch_model-00010-of-00012.bin",
+    "gpt_neox.layers.27.attention.rotary_emb.inv_freq": "pytorch_model-00010-of-00012.bin",
+    "gpt_neox.layers.27.input_layernorm.bias": "pytorch_model-00010-of-00012.bin",
+    "gpt_neox.layers.27.input_layernorm.weight": "pytorch_model-00010-of-00012.bin",
+    "gpt_neox.layers.27.mlp.dense_4h_to_h.bias": "pytorch_model-00010-of-00012.bin",
+    "gpt_neox.layers.27.mlp.dense_4h_to_h.weight": "pytorch_model-00010-of-00012.bin",
+    "gpt_neox.layers.27.mlp.dense_h_to_4h.bias": "pytorch_model-00010-of-00012.bin",
+    "gpt_neox.layers.27.mlp.dense_h_to_4h.weight": "pytorch_model-00010-of-00012.bin",
+    "gpt_neox.layers.27.post_attention_layernorm.bias": "pytorch_model-00010-of-00012.bin",
+    "gpt_neox.layers.27.post_attention_layernorm.weight": "pytorch_model-00010-of-00012.bin",
+    "gpt_neox.layers.28.attention.bias": "pytorch_model-00010-of-00012.bin",
+    "gpt_neox.layers.28.attention.dense.bias": "pytorch_model-00010-of-00012.bin",
+    "gpt_neox.layers.28.attention.dense.weight": "pytorch_model-00010-of-00012.bin",
+    "gpt_neox.layers.28.attention.masked_bias": "pytorch_model-00010-of-00012.bin",
+    "gpt_neox.layers.28.attention.query_key_value.bias": "pytorch_model-00010-of-00012.bin",
+    "gpt_neox.layers.28.attention.query_key_value.weight": "pytorch_model-00010-of-00012.bin",
+    "gpt_neox.layers.28.attention.rotary_emb.inv_freq": "pytorch_model-00010-of-00012.bin",
+    "gpt_neox.layers.28.input_layernorm.bias": "pytorch_model-00010-of-00012.bin",
+    "gpt_neox.layers.28.input_layernorm.weight": "pytorch_model-00010-of-00012.bin",
+    "gpt_neox.layers.28.mlp.dense_4h_to_h.bias": "pytorch_model-00011-of-00012.bin",
+    "gpt_neox.layers.28.mlp.dense_4h_to_h.weight": "pytorch_model-00011-of-00012.bin",
+    "gpt_neox.layers.28.mlp.dense_h_to_4h.bias": "pytorch_model-00011-of-00012.bin",
+    "gpt_neox.layers.28.mlp.dense_h_to_4h.weight": "pytorch_model-00011-of-00012.bin",
+    "gpt_neox.layers.28.post_attention_layernorm.bias": "pytorch_model-00010-of-00012.bin",
+    "gpt_neox.layers.28.post_attention_layernorm.weight": "pytorch_model-00010-of-00012.bin",
+    "gpt_neox.layers.29.attention.bias": "pytorch_model-00011-of-00012.bin",
+    "gpt_neox.layers.29.attention.dense.bias": "pytorch_model-00011-of-00012.bin",
+    "gpt_neox.layers.29.attention.dense.weight": "pytorch_model-00011-of-00012.bin",
+    "gpt_neox.layers.29.attention.masked_bias": "pytorch_model-00011-of-00012.bin",
+    "gpt_neox.layers.29.attention.query_key_value.bias": "pytorch_model-00011-of-00012.bin",
+    "gpt_neox.layers.29.attention.query_key_value.weight": "pytorch_model-00011-of-00012.bin",
+    "gpt_neox.layers.29.attention.rotary_emb.inv_freq": "pytorch_model-00011-of-00012.bin",
+    "gpt_neox.layers.29.input_layernorm.bias": "pytorch_model-00011-of-00012.bin",
+    "gpt_neox.layers.29.input_layernorm.weight": "pytorch_model-00011-of-00012.bin",
+    "gpt_neox.layers.29.mlp.dense_4h_to_h.bias": "pytorch_model-00011-of-00012.bin",
+    "gpt_neox.layers.29.mlp.dense_4h_to_h.weight": "pytorch_model-00011-of-00012.bin",
+    "gpt_neox.layers.29.mlp.dense_h_to_4h.bias": "pytorch_model-00011-of-00012.bin",
+    "gpt_neox.layers.29.mlp.dense_h_to_4h.weight": "pytorch_model-00011-of-00012.bin",
+    "gpt_neox.layers.29.post_attention_layernorm.bias": "pytorch_model-00011-of-00012.bin",
+    "gpt_neox.layers.29.post_attention_layernorm.weight": "pytorch_model-00011-of-00012.bin",
+    "gpt_neox.layers.3.attention.bias": "pytorch_model-00002-of-00012.bin",
+    "gpt_neox.layers.3.attention.dense.bias": "pytorch_model-00002-of-00012.bin",
+    "gpt_neox.layers.3.attention.dense.weight": "pytorch_model-00002-of-00012.bin",
+    "gpt_neox.layers.3.attention.masked_bias": "pytorch_model-00002-of-00012.bin",
+    "gpt_neox.layers.3.attention.query_key_value.bias": "pytorch_model-00002-of-00012.bin",
+    "gpt_neox.layers.3.attention.query_key_value.weight": "pytorch_model-00002-of-00012.bin",
+    "gpt_neox.layers.3.attention.rotary_emb.inv_freq": "pytorch_model-00002-of-00012.bin",
+    "gpt_neox.layers.3.input_layernorm.bias": "pytorch_model-00002-of-00012.bin",
+    "gpt_neox.layers.3.input_layernorm.weight": "pytorch_model-00002-of-00012.bin",
+    "gpt_neox.layers.3.mlp.dense_4h_to_h.bias": "pytorch_model-00002-of-00012.bin",
+    "gpt_neox.layers.3.mlp.dense_4h_to_h.weight": "pytorch_model-00002-of-00012.bin",
+    "gpt_neox.layers.3.mlp.dense_h_to_4h.bias": "pytorch_model-00002-of-00012.bin",
+    "gpt_neox.layers.3.mlp.dense_h_to_4h.weight": "pytorch_model-00002-of-00012.bin",
+    "gpt_neox.layers.3.post_attention_layernorm.bias": "pytorch_model-00002-of-00012.bin",
+    "gpt_neox.layers.3.post_attention_layernorm.weight": "pytorch_model-00002-of-00012.bin",
+    "gpt_neox.layers.30.attention.bias": "pytorch_model-00011-of-00012.bin",
+    "gpt_neox.layers.30.attention.dense.bias": "pytorch_model-00011-of-00012.bin",
+    "gpt_neox.layers.30.attention.dense.weight": "pytorch_model-00011-of-00012.bin",
+    "gpt_neox.layers.30.attention.masked_bias": "pytorch_model-00011-of-00012.bin",
+    "gpt_neox.layers.30.attention.query_key_value.bias": "pytorch_model-00011-of-00012.bin",
+    "gpt_neox.layers.30.attention.query_key_value.weight": "pytorch_model-00011-of-00012.bin",
+    "gpt_neox.layers.30.attention.rotary_emb.inv_freq": "pytorch_model-00011-of-00012.bin",
+    "gpt_neox.layers.30.input_layernorm.bias": "pytorch_model-00011-of-00012.bin",
+    "gpt_neox.layers.30.input_layernorm.weight": "pytorch_model-00011-of-00012.bin",
+    "gpt_neox.layers.30.mlp.dense_4h_to_h.bias": "pytorch_model-00011-of-00012.bin",
+    "gpt_neox.layers.30.mlp.dense_4h_to_h.weight": "pytorch_model-00011-of-00012.bin",
+    "gpt_neox.layers.30.mlp.dense_h_to_4h.bias": "pytorch_model-00011-of-00012.bin",
+    "gpt_neox.layers.30.mlp.dense_h_to_4h.weight": "pytorch_model-00011-of-00012.bin",
+    "gpt_neox.layers.30.post_attention_layernorm.bias": "pytorch_model-00011-of-00012.bin",
+    "gpt_neox.layers.30.post_attention_layernorm.weight": "pytorch_model-00011-of-00012.bin",
+    "gpt_neox.layers.31.attention.bias": "pytorch_model-00011-of-00012.bin",
+    "gpt_neox.layers.31.attention.dense.bias": "pytorch_model-00011-of-00012.bin",
+    "gpt_neox.layers.31.attention.dense.weight": "pytorch_model-00011-of-00012.bin",
+    "gpt_neox.layers.31.attention.masked_bias": "pytorch_model-00011-of-00012.bin",
+    "gpt_neox.layers.31.attention.query_key_value.bias": "pytorch_model-00011-of-00012.bin",
+    "gpt_neox.layers.31.attention.query_key_value.weight": "pytorch_model-00011-of-00012.bin",
+    "gpt_neox.layers.31.attention.rotary_emb.inv_freq": "pytorch_model-00011-of-00012.bin",
+    "gpt_neox.layers.31.input_layernorm.bias": "pytorch_model-00011-of-00012.bin",
+    "gpt_neox.layers.31.input_layernorm.weight": "pytorch_model-00011-of-00012.bin",
+    "gpt_neox.layers.31.mlp.dense_4h_to_h.bias": "pytorch_model-00012-of-00012.bin",
+    "gpt_neox.layers.31.mlp.dense_4h_to_h.weight": "pytorch_model-00012-of-00012.bin",
+    "gpt_neox.layers.31.mlp.dense_h_to_4h.bias": "pytorch_model-00012-of-00012.bin",
+    "gpt_neox.layers.31.mlp.dense_h_to_4h.weight": "pytorch_model-00012-of-00012.bin",
+    "gpt_neox.layers.31.post_attention_layernorm.bias": "pytorch_model-00011-of-00012.bin",
+    "gpt_neox.layers.31.post_attention_layernorm.weight": "pytorch_model-00011-of-00012.bin",
+    "gpt_neox.layers.4.attention.bias": "pytorch_model-00002-of-00012.bin",
+    "gpt_neox.layers.4.attention.dense.bias": "pytorch_model-00002-of-00012.bin",
+    "gpt_neox.layers.4.attention.dense.weight": "pytorch_model-00002-of-00012.bin",
+    "gpt_neox.layers.4.attention.masked_bias": "pytorch_model-00002-of-00012.bin",
+    "gpt_neox.layers.4.attention.query_key_value.bias": "pytorch_model-00002-of-00012.bin",
+    "gpt_neox.layers.4.attention.query_key_value.weight": "pytorch_model-00002-of-00012.bin",
+    "gpt_neox.layers.4.attention.rotary_emb.inv_freq": "pytorch_model-00002-of-00012.bin",
+    "gpt_neox.layers.4.input_layernorm.bias": "pytorch_model-00002-of-00012.bin",
+    "gpt_neox.layers.4.input_layernorm.weight": "pytorch_model-00002-of-00012.bin",
+    "gpt_neox.layers.4.mlp.dense_4h_to_h.bias": "pytorch_model-00003-of-00012.bin",
+    "gpt_neox.layers.4.mlp.dense_4h_to_h.weight": "pytorch_model-00003-of-00012.bin",
+    "gpt_neox.layers.4.mlp.dense_h_to_4h.bias": "pytorch_model-00003-of-00012.bin",
+    "gpt_neox.layers.4.mlp.dense_h_to_4h.weight": "pytorch_model-00003-of-00012.bin",
+    "gpt_neox.layers.4.post_attention_layernorm.bias": "pytorch_model-00002-of-00012.bin",
+    "gpt_neox.layers.4.post_attention_layernorm.weight": "pytorch_model-00002-of-00012.bin",
+    "gpt_neox.layers.5.attention.bias": "pytorch_model-00003-of-00012.bin",
+    "gpt_neox.layers.5.attention.dense.bias": "pytorch_model-00003-of-00012.bin",
+    "gpt_neox.layers.5.attention.dense.weight": "pytorch_model-00003-of-00012.bin",
+    "gpt_neox.layers.5.attention.masked_bias": "pytorch_model-00003-of-00012.bin",
+    "gpt_neox.layers.5.attention.query_key_value.bias": "pytorch_model-00003-of-00012.bin",
+    "gpt_neox.layers.5.attention.query_key_value.weight": "pytorch_model-00003-of-00012.bin",
+    "gpt_neox.layers.5.attention.rotary_emb.inv_freq": "pytorch_model-00003-of-00012.bin",
+    "gpt_neox.layers.5.input_layernorm.bias": "pytorch_model-00003-of-00012.bin",
+    "gpt_neox.layers.5.input_layernorm.weight": "pytorch_model-00003-of-00012.bin",
+    "gpt_neox.layers.5.mlp.dense_4h_to_h.bias": "pytorch_model-00003-of-00012.bin",
+    "gpt_neox.layers.5.mlp.dense_4h_to_h.weight": "pytorch_model-00003-of-00012.bin",
+    "gpt_neox.layers.5.mlp.dense_h_to_4h.bias": "pytorch_model-00003-of-00012.bin",
+    "gpt_neox.layers.5.mlp.dense_h_to_4h.weight": "pytorch_model-00003-of-00012.bin",
+    "gpt_neox.layers.5.post_attention_layernorm.bias": "pytorch_model-00003-of-00012.bin",
+    "gpt_neox.layers.5.post_attention_layernorm.weight": "pytorch_model-00003-of-00012.bin",
+    "gpt_neox.layers.6.attention.bias": "pytorch_model-00003-of-00012.bin",
+    "gpt_neox.layers.6.attention.dense.bias": "pytorch_model-00003-of-00012.bin",
+    "gpt_neox.layers.6.attention.dense.weight": "pytorch_model-00003-of-00012.bin",
+    "gpt_neox.layers.6.attention.masked_bias": "pytorch_model-00003-of-00012.bin",
+    "gpt_neox.layers.6.attention.query_key_value.bias": "pytorch_model-00003-of-00012.bin",
+    "gpt_neox.layers.6.attention.query_key_value.weight": "pytorch_model-00003-of-00012.bin",
+    "gpt_neox.layers.6.attention.rotary_emb.inv_freq": "pytorch_model-00003-of-00012.bin",
+    "gpt_neox.layers.6.input_layernorm.bias": "pytorch_model-00003-of-00012.bin",
+    "gpt_neox.layers.6.input_layernorm.weight": "pytorch_model-00003-of-00012.bin",
+    "gpt_neox.layers.6.mlp.dense_4h_to_h.bias": "pytorch_model-00003-of-00012.bin",
+    "gpt_neox.layers.6.mlp.dense_4h_to_h.weight": "pytorch_model-00003-of-00012.bin",
+    "gpt_neox.layers.6.mlp.dense_h_to_4h.bias": "pytorch_model-00003-of-00012.bin",
+    "gpt_neox.layers.6.mlp.dense_h_to_4h.weight": "pytorch_model-00003-of-00012.bin",
+    "gpt_neox.layers.6.post_attention_layernorm.bias": "pytorch_model-00003-of-00012.bin",
+    "gpt_neox.layers.6.post_attention_layernorm.weight": "pytorch_model-00003-of-00012.bin",
+    "gpt_neox.layers.7.attention.bias": "pytorch_model-00003-of-00012.bin",
+    "gpt_neox.layers.7.attention.dense.bias": "pytorch_model-00003-of-00012.bin",
+    "gpt_neox.layers.7.attention.dense.weight": "pytorch_model-00003-of-00012.bin",
+    "gpt_neox.layers.7.attention.masked_bias": "pytorch_model-00003-of-00012.bin",
+    "gpt_neox.layers.7.attention.query_key_value.bias": "pytorch_model-00003-of-00012.bin",
+    "gpt_neox.layers.7.attention.query_key_value.weight": "pytorch_model-00003-of-00012.bin",
+    "gpt_neox.layers.7.attention.rotary_emb.inv_freq": "pytorch_model-00003-of-00012.bin",
+    "gpt_neox.layers.7.input_layernorm.bias": "pytorch_model-00003-of-00012.bin",
+    "gpt_neox.layers.7.input_layernorm.weight": "pytorch_model-00003-of-00012.bin",
+    "gpt_neox.layers.7.mlp.dense_4h_to_h.bias": "pytorch_model-00004-of-00012.bin",
+    "gpt_neox.layers.7.mlp.dense_4h_to_h.weight": "pytorch_model-00004-of-00012.bin",
+    "gpt_neox.layers.7.mlp.dense_h_to_4h.bias": "pytorch_model-00004-of-00012.bin",
+    "gpt_neox.layers.7.mlp.dense_h_to_4h.weight": "pytorch_model-00004-of-00012.bin",
+    "gpt_neox.layers.7.post_attention_layernorm.bias": "pytorch_model-00003-of-00012.bin",
+    "gpt_neox.layers.7.post_attention_layernorm.weight": "pytorch_model-00003-of-00012.bin",
+    "gpt_neox.layers.8.attention.bias": "pytorch_model-00004-of-00012.bin",
+    "gpt_neox.layers.8.attention.dense.bias": "pytorch_model-00004-of-00012.bin",
+    "gpt_neox.layers.8.attention.dense.weight": "pytorch_model-00004-of-00012.bin",
+    "gpt_neox.layers.8.attention.masked_bias": "pytorch_model-00004-of-00012.bin",
+    "gpt_neox.layers.8.attention.query_key_value.bias": "pytorch_model-00004-of-00012.bin",
+    "gpt_neox.layers.8.attention.query_key_value.weight": "pytorch_model-00004-of-00012.bin",
+    "gpt_neox.layers.8.attention.rotary_emb.inv_freq": "pytorch_model-00004-of-00012.bin",
+    "gpt_neox.layers.8.input_layernorm.bias": "pytorch_model-00004-of-00012.bin",
+    "gpt_neox.layers.8.input_layernorm.weight": "pytorch_model-00004-of-00012.bin",
+    "gpt_neox.layers.8.mlp.dense_4h_to_h.bias": "pytorch_model-00004-of-00012.bin",
+    "gpt_neox.layers.8.mlp.dense_4h_to_h.weight": "pytorch_model-00004-of-00012.bin",
+    "gpt_neox.layers.8.mlp.dense_h_to_4h.bias": "pytorch_model-00004-of-00012.bin",
+    "gpt_neox.layers.8.mlp.dense_h_to_4h.weight": "pytorch_model-00004-of-00012.bin",
+    "gpt_neox.layers.8.post_attention_layernorm.bias": "pytorch_model-00004-of-00012.bin",
+    "gpt_neox.layers.8.post_attention_layernorm.weight": "pytorch_model-00004-of-00012.bin",
+    "gpt_neox.layers.9.attention.bias": "pytorch_model-00004-of-00012.bin",
+    "gpt_neox.layers.9.attention.dense.bias": "pytorch_model-00004-of-00012.bin",
+    "gpt_neox.layers.9.attention.dense.weight": "pytorch_model-00004-of-00012.bin",
+    "gpt_neox.layers.9.attention.masked_bias": "pytorch_model-00004-of-00012.bin",
+    "gpt_neox.layers.9.attention.query_key_value.bias": "pytorch_model-00004-of-00012.bin",
+    "gpt_neox.layers.9.attention.query_key_value.weight": "pytorch_model-00004-of-00012.bin",
+    "gpt_neox.layers.9.attention.rotary_emb.inv_freq": "pytorch_model-00004-of-00012.bin",
+    "gpt_neox.layers.9.input_layernorm.bias": "pytorch_model-00004-of-00012.bin",
+    "gpt_neox.layers.9.input_layernorm.weight": "pytorch_model-00004-of-00012.bin",
+    "gpt_neox.layers.9.mlp.dense_4h_to_h.bias": "pytorch_model-00004-of-00012.bin",
+    "gpt_neox.layers.9.mlp.dense_4h_to_h.weight": "pytorch_model-00004-of-00012.bin",
+    "gpt_neox.layers.9.mlp.dense_h_to_4h.bias": "pytorch_model-00004-of-00012.bin",
+    "gpt_neox.layers.9.mlp.dense_h_to_4h.weight": "pytorch_model-00004-of-00012.bin",
+    "gpt_neox.layers.9.post_attention_layernorm.bias": "pytorch_model-00004-of-00012.bin",
+    "gpt_neox.layers.9.post_attention_layernorm.weight": "pytorch_model-00004-of-00012.bin"
+  }
+}
diff --git a/special_tokens_map.json b/special_tokens_map.json
new file mode 100644
index 0000000..0204ed1
--- /dev/null
+++ b/special_tokens_map.json
@@ -0,0 +1,5 @@
+{
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "unk_token": "<|endoftext|>"
+}
diff --git a/tokenizer.json b/tokenizer.json
new file mode 100644
index 0000000..8fa6a67
--- /dev/null
+++ b/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c24618a1b3e6a38167beff1c72cffd126c3a66254347304b50547d12c5f25624
+size 2113710
diff --git a/tokenizer_config.json b/tokenizer_config.json
new file mode 100644
index 0000000..512beb8
--- /dev/null
+++ b/tokenizer_config.json
@@ -0,0 +1,9 @@
+{
+  "add_prefix_space": false,
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "model_max_length": 1000000000000000019884624838656,
+  "special_tokens_map_file": "/fsx/home-guac/.cache/huggingface/transformers/models--EleutherAI--gpt-neox-20b/snapshots/4e49eadb5d14bd22f314ec3f45b69a87b88c7691/special_tokens_map.json",
+  "tokenizer_class": "GPTNeoXTokenizer",
+  "unk_token": "<|endoftext|>"
+}