commit 73e7e75ce48f51b0d51e6dadfee3f6532ae2b096 Author: ModelHub XC Date: Tue Jul 28 08:07:05 2026 +0800 初始化项目，由ModelHub XC社区提供模型 Model: togethercomputer/RedPajama-INCITE-7B-Chat Source: Original Platform diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..ec20114 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,51 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text + + +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text + +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zstandard filter=lfs diff=lfs merge=lfs -text +*.tfevents* filter=lfs diff=lfs merge=lfs -text +*.db* filter=lfs diff=lfs merge=lfs -text +*.ark* filter=lfs diff=lfs merge=lfs -text +**/*ckpt*data* filter=lfs diff=lfs merge=lfs -text +**/*ckpt*.meta filter=lfs diff=lfs merge=lfs -text +**/*ckpt*.index filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.gguf* filter=lfs diff=lfs merge=lfs -text +*.ggml filter=lfs diff=lfs merge=lfs -text +*.llamafile* filter=lfs diff=lfs merge=lfs -text +*.pt2 filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text + +pytorch_model-00001-of-00002.bin filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text +pytorch_model-00002-of-00002.bin filter=lfs diff=lfs merge=lfs -text \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..f769baa --- /dev/null +++ b/README.md @@ -0,0 +1,213 @@ +--- +license: apache-2.0 +language: +- en +datasets: +- togethercomputer/RedPajama-Data-1T +- OpenAssistant/oasst1 +- databricks/databricks-dolly-15k +widget: +- text: ": Write an email to my friends inviting them to come to my home on Friday for a dinner party, bring their own food to share.\n:" + example_title: "Email Writing" +- text: ": Create a list of things to do in San Francisco\n:" + example_title: "Brainstorming" +inference: + parameters: + temperature: 0.7 + top_p: 0.7 + top_k: 50 + max_new_tokens: 128 +--- + +# RedPajama-INCITE-7B-Chat + +RedPajama-INCITE-7B-Chat was developed by Together and leaders from the open-source AI community including Ontocord.ai, ETH DS3Lab, AAI CERC, Université de Montréal, MILA - Québec AI Institute, Stanford Center for Research on Foundation Models (CRFM), Stanford Hazy Research research group and LAION. + +It is fine-tuned on OASST1 and Dolly2 to enhance chatting ability. + + - Base Model: [RedPajama-INCITE-7B-Base](https://huggingface.co/togethercomputer/RedPajama-INCITE-7B-Base) + - Instruction-tuned Version: [RedPajama-INCITE-7B-Instruct](https://huggingface.co/togethercomputer/RedPajama-INCITE-7B-Instruct) + - Chat Version: [RedPajama-INCITE-7B-Chat](https://huggingface.co/togethercomputer/RedPajama-INCITE-7B-Chat) + + +## Model Details +- **Developed by**: Together Computer. +- **Model type**: Language Model +- **Language(s)**: English +- **License**: Apache 2.0 +- **Model Description**: A 6.9B parameter pretrained language model. + +# Quick Start + +Please note that the model requires `transformers` version >= 4.25.1. + +To prompt the chat model, use the following format: +``` +: [Instruction] +: +``` + +## GPU Inference + +This requires a GPU with 16GB memory. + +```python +import torch +import transformers +from transformers import AutoTokenizer, AutoModelForCausalLM + +MIN_TRANSFORMERS_VERSION = '4.25.1' + +# check transformers version +assert transformers.__version__ >= MIN_TRANSFORMERS_VERSION, f'Please upgrade transformers to version {MIN_TRANSFORMERS_VERSION} or higher.' + +# init +tokenizer = AutoTokenizer.from_pretrained("togethercomputer/RedPajama-INCITE-7B-Chat") +model = AutoModelForCausalLM.from_pretrained("togethercomputer/RedPajama-INCITE-7B-Chat", torch_dtype=torch.float16) +model = model.to('cuda:0') +# infer +prompt = ": Who is Alan Turing?\n:" +inputs = tokenizer(prompt, return_tensors='pt').to(model.device) +input_length = inputs.input_ids.shape[1] +outputs = model.generate( + **inputs, max_new_tokens=128, do_sample=True, temperature=0.7, top_p=0.7, top_k=50, return_dict_in_generate=True +) +token = outputs.sequences[0, input_length:] +output_str = tokenizer.decode(token) +print(output_str) +""" +Alan Mathison Turing (23 June 1912 7 June 1954) was an English computer scientist, mathematician, logician, cryptanalyst, philosopher, mathematician, and theoretical biologist. +""" +``` + +## GPU Inference in Int8 + +This requires a GPU with 12GB memory. + +To run inference with int8, please ensure you have installed accelerate and bitandbytes. You can install them with the following command: + +```bash +pip install accelerate +pip install bitsandbytes +``` + +Then you can run inference with int8 as follows: + +```python +import torch +import transformers +from transformers import AutoTokenizer, AutoModelForCausalLM + +MIN_TRANSFORMERS_VERSION = '4.25.1' + +# check transformers version +assert transformers.__version__ >= MIN_TRANSFORMERS_VERSION, f'Please upgrade transformers to version {MIN_TRANSFORMERS_VERSION} or higher.' + +# init +tokenizer = AutoTokenizer.from_pretrained("togethercomputer/RedPajama-INCITE-7B-Chat") +model = AutoModelForCausalLM.from_pretrained("togethercomputer/RedPajama-INCITE-7B-Chat", device_map='auto', torch_dtype=torch.float16, load_in_8bit=True) + +# infer +prompt = ": Who is Alan Turing?\n:" +inputs = tokenizer(prompt, return_tensors='pt').to(model.device) +input_length = inputs.input_ids.shape[1] +outputs = model.generate( + **inputs, max_new_tokens=128, do_sample=True, temperature=0.7, top_p=0.7, top_k=50, return_dict_in_generate=True +) +token = outputs.sequences[0, input_length:] +output_str = tokenizer.decode(token) +print(output_str) +""" +Alan Mathison Turing (23 June 1912 – 7 June 1954) was an English computer scientist, mathematician, logician, cryptanalyst, philosopher, and theoretical biologist. +""" +``` + +## CPU Inference + +```python +import torch +import transformers +from transformers import AutoTokenizer, AutoModelForCausalLM + +MIN_TRANSFORMERS_VERSION = '4.25.1' + +# check transformers version +assert transformers.__version__ >= MIN_TRANSFORMERS_VERSION, f'Please upgrade transformers to version {MIN_TRANSFORMERS_VERSION} or higher.' + +# init +tokenizer = AutoTokenizer.from_pretrained("togethercomputer/RedPajama-INCITE-7B-Chat") +model = AutoModelForCausalLM.from_pretrained("togethercomputer/RedPajama-INCITE-7B-Chat", torch_dtype=torch.bfloat16) +# infer +prompt = ": Who is Alan Turing?\n:" +inputs = tokenizer(prompt, return_tensors='pt').to(model.device) +input_length = inputs.input_ids.shape[1] +outputs = model.generate( + **inputs, max_new_tokens=128, do_sample=True, temperature=0.7, top_p=0.7, top_k=50, return_dict_in_generate=True +) +token = outputs.sequences[0, input_length:] +output_str = tokenizer.decode(token) +print(output_str) +""" +Alan Mathison Turing, OBE, FRS, (23 June 1912 – 7 June 1954) was an English computer scientist, mathematician, logician, cryptanalyst, philosopher, and theoretical biologist. +""" +``` + +Please note that since `LayerNormKernelImpl` is not implemented in fp16 for CPU, we use `bfloat16` for CPU inference. + + +# Uses + +## Direct Use + +Excluded uses are described below. + +### Misuse, Malicious Use, and Out-of-Scope Use + +It is the responsibility of the end user to ensure that the model is used in a responsible and ethical manner. + +#### Out-of-Scope Use + +`RedPajama-INCITE-7B-Chat` is a language model and may not perform well for other use cases outside of its intended scope. +For example, it may not be suitable for use in safety-critical applications or for making decisions that have a significant impact on individuals or society. +It is important to consider the limitations of the model and to only use it for its intended purpose. + +#### Misuse and Malicious Use + +`RedPajama-INCITE-7B-Chat` is designed for language modeling. +Misuse of the model, such as using it to engage in illegal or unethical activities, is strictly prohibited and goes against the principles of the project. + +Using the model to generate content that is cruel to individuals is a misuse of this model. This includes, but is not limited to: + +- Generating fake news, misinformation, or propaganda +- Promoting hate speech, discrimination, or violence against individuals or groups +- Impersonating individuals or organizations without their consent +- Engaging in cyberbullying or harassment +- Defamatory content +- Spamming or scamming +- Sharing confidential or sensitive information without proper authorization +- Violating the terms of use of the model or the data used to train it +- Creating automated bots for malicious purposes such as spreading malware, phishing scams, or spamming + +## Limitations + +`RedPajama-INCITE-7B-Chat`, like other language models, has limitations that should be taken into consideration. +For example, the model may not always provide accurate or relevant answers, particularly for questions that are complex, ambiguous, or outside of its training data. +We therefore welcome contributions from individuals and organizations, and encourage collaboration towards creating a more robust and inclusive chatbot. + +## Training + +**Training Data** + +Please refer to [togethercomputer/RedPajama-Data-1T](https://huggingface.co/datasets/togethercomputer/RedPajama-Data-1T) + +**Training Procedure** + +- **Hardware:** 8 A100 +- **Optimizer:** Adam +- **Gradient Accumulations**: 1 +- **Num of Tokens:** 79M tokens +- **Learning rate:** 1e-5 + +## Community + +Join us on [Together Discord](https://discord.gg/6ZVDU8tTD4) \ No newline at end of file diff --git a/config.json b/config.json new file mode 100644 index 0000000..08eee39 --- /dev/null +++ b/config.json @@ -0,0 +1,25 @@ +{ + "_name_or_path": "togethercomputer/RedPajama-INCITE-Chat-7B-v1", + "architectures": [ + "GPTNeoXForCausalLM" + ], + "bos_token_id": 0, + "eos_token_id": 0, + "hidden_act": "gelu", + "hidden_size": 4096, + "initializer_range": 0.02, + "intermediate_size": 16384, + "layer_norm_eps": 1e-05, + "max_position_embeddings": 2048, + "model_type": "gpt_neox", + "num_attention_heads": 32, + "num_hidden_layers": 32, + "rotary_emb_base": 10000, + "rotary_pct": 1.0, + "tie_word_embeddings": false, + "torch_dtype": "float16", + "transformers_version": "4.28.1", + "use_cache": true, + "use_parallel_residual": false, + "vocab_size": 50432 +} diff --git a/configuration.json b/configuration.json new file mode 100644 index 0000000..bbeeda1 --- /dev/null +++ b/configuration.json @@ -0,0 +1 @@ +{"framework": "pytorch", "task": "text-generation", "allow_remote": true} \ No newline at end of file diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..a2ee315 --- /dev/null +++ b/generation_config.json @@ -0,0 +1,6 @@ +{ + "_from_model_config": true, + "bos_token_id": 0, + "eos_token_id": 0, + "transformers_version": "4.29.1" +} diff --git a/pytorch_model-00001-of-00002.bin b/pytorch_model-00001-of-00002.bin new file mode 100644 index 0000000..204c127 --- /dev/null +++ b/pytorch_model-00001-of-00002.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0df1e151ee1d94b0e29e432a4fb0648b0bf1440db13020756eba87677c6611be +size 10045936084 diff --git a/pytorch_model-00002-of-00002.bin b/pytorch_model-00002-of-00002.bin new file mode 100644 index 0000000..beb91ea --- /dev/null +++ b/pytorch_model-00002-of-00002.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b92d9c8a21a1530a23c65a07e5645df7d410555a0421531e1da94cd08d7c0410 +size 3803056370 diff --git a/pytorch_model.bin.index.json b/pytorch_model.bin.index.json new file mode 100644 index 0000000..9f60147 --- /dev/null +++ b/pytorch_model.bin.index.json @@ -0,0 +1,491 @@ +{ + "metadata": { + "total_size": 13731385408.0 + }, + "weight_map": { + "embed_out.weight": "pytorch_model-00002-of-00002.bin", + "gpt_neox.embed_in.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.final_layer_norm.bias": "pytorch_model-00002-of-00002.bin", + "gpt_neox.final_layer_norm.weight": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.0.attention.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.0.attention.dense.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.0.attention.dense.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.0.attention.masked_bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.0.attention.query_key_value.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.0.attention.query_key_value.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.0.attention.rotary_emb.inv_freq": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.0.input_layernorm.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.0.input_layernorm.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.0.mlp.dense_4h_to_h.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.0.mlp.dense_4h_to_h.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.0.mlp.dense_h_to_4h.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.0.mlp.dense_h_to_4h.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.0.post_attention_layernorm.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.0.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.1.attention.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.1.attention.dense.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.1.attention.dense.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.1.attention.masked_bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.1.attention.query_key_value.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.1.attention.query_key_value.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.1.attention.rotary_emb.inv_freq": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.1.input_layernorm.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.1.input_layernorm.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.1.mlp.dense_4h_to_h.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.1.mlp.dense_4h_to_h.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.1.mlp.dense_h_to_4h.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.1.mlp.dense_h_to_4h.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.1.post_attention_layernorm.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.1.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.10.attention.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.10.attention.dense.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.10.attention.dense.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.10.attention.masked_bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.10.attention.query_key_value.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.10.attention.query_key_value.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.10.attention.rotary_emb.inv_freq": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.10.input_layernorm.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.10.input_layernorm.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.10.mlp.dense_4h_to_h.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.10.mlp.dense_4h_to_h.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.10.mlp.dense_h_to_4h.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.10.mlp.dense_h_to_4h.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.10.post_attention_layernorm.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.10.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.11.attention.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.11.attention.dense.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.11.attention.dense.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.11.attention.masked_bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.11.attention.query_key_value.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.11.attention.query_key_value.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.11.attention.rotary_emb.inv_freq": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.11.input_layernorm.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.11.input_layernorm.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.11.mlp.dense_4h_to_h.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.11.mlp.dense_4h_to_h.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.11.mlp.dense_h_to_4h.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.11.mlp.dense_h_to_4h.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.11.post_attention_layernorm.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.11.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.12.attention.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.12.attention.dense.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.12.attention.dense.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.12.attention.masked_bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.12.attention.query_key_value.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.12.attention.query_key_value.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.12.attention.rotary_emb.inv_freq": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.12.input_layernorm.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.12.input_layernorm.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.12.mlp.dense_4h_to_h.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.12.mlp.dense_4h_to_h.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.12.mlp.dense_h_to_4h.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.12.mlp.dense_h_to_4h.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.12.post_attention_layernorm.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.12.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.13.attention.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.13.attention.dense.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.13.attention.dense.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.13.attention.masked_bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.13.attention.query_key_value.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.13.attention.query_key_value.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.13.attention.rotary_emb.inv_freq": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.13.input_layernorm.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.13.input_layernorm.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.13.mlp.dense_4h_to_h.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.13.mlp.dense_4h_to_h.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.13.mlp.dense_h_to_4h.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.13.mlp.dense_h_to_4h.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.13.post_attention_layernorm.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.13.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.14.attention.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.14.attention.dense.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.14.attention.dense.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.14.attention.masked_bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.14.attention.query_key_value.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.14.attention.query_key_value.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.14.attention.rotary_emb.inv_freq": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.14.input_layernorm.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.14.input_layernorm.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.14.mlp.dense_4h_to_h.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.14.mlp.dense_4h_to_h.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.14.mlp.dense_h_to_4h.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.14.mlp.dense_h_to_4h.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.14.post_attention_layernorm.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.14.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.15.attention.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.15.attention.dense.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.15.attention.dense.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.15.attention.masked_bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.15.attention.query_key_value.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.15.attention.query_key_value.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.15.attention.rotary_emb.inv_freq": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.15.input_layernorm.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.15.input_layernorm.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.15.mlp.dense_4h_to_h.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.15.mlp.dense_4h_to_h.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.15.mlp.dense_h_to_4h.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.15.mlp.dense_h_to_4h.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.15.post_attention_layernorm.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.15.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.16.attention.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.16.attention.dense.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.16.attention.dense.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.16.attention.masked_bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.16.attention.query_key_value.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.16.attention.query_key_value.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.16.attention.rotary_emb.inv_freq": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.16.input_layernorm.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.16.input_layernorm.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.16.mlp.dense_4h_to_h.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.16.mlp.dense_4h_to_h.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.16.mlp.dense_h_to_4h.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.16.mlp.dense_h_to_4h.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.16.post_attention_layernorm.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.16.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.17.attention.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.17.attention.dense.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.17.attention.dense.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.17.attention.masked_bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.17.attention.query_key_value.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.17.attention.query_key_value.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.17.attention.rotary_emb.inv_freq": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.17.input_layernorm.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.17.input_layernorm.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.17.mlp.dense_4h_to_h.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.17.mlp.dense_4h_to_h.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.17.mlp.dense_h_to_4h.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.17.mlp.dense_h_to_4h.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.17.post_attention_layernorm.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.17.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.18.attention.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.18.attention.dense.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.18.attention.dense.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.18.attention.masked_bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.18.attention.query_key_value.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.18.attention.query_key_value.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.18.attention.rotary_emb.inv_freq": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.18.input_layernorm.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.18.input_layernorm.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.18.mlp.dense_4h_to_h.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.18.mlp.dense_4h_to_h.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.18.mlp.dense_h_to_4h.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.18.mlp.dense_h_to_4h.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.18.post_attention_layernorm.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.18.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.19.attention.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.19.attention.dense.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.19.attention.dense.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.19.attention.masked_bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.19.attention.query_key_value.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.19.attention.query_key_value.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.19.attention.rotary_emb.inv_freq": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.19.input_layernorm.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.19.input_layernorm.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.19.mlp.dense_4h_to_h.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.19.mlp.dense_4h_to_h.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.19.mlp.dense_h_to_4h.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.19.mlp.dense_h_to_4h.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.19.post_attention_layernorm.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.19.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.2.attention.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.2.attention.dense.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.2.attention.dense.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.2.attention.masked_bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.2.attention.query_key_value.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.2.attention.query_key_value.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.2.attention.rotary_emb.inv_freq": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.2.input_layernorm.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.2.input_layernorm.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.2.mlp.dense_4h_to_h.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.2.mlp.dense_4h_to_h.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.2.mlp.dense_h_to_4h.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.2.mlp.dense_h_to_4h.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.2.post_attention_layernorm.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.2.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.20.attention.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.20.attention.dense.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.20.attention.dense.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.20.attention.masked_bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.20.attention.query_key_value.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.20.attention.query_key_value.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.20.attention.rotary_emb.inv_freq": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.20.input_layernorm.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.20.input_layernorm.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.20.mlp.dense_4h_to_h.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.20.mlp.dense_4h_to_h.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.20.mlp.dense_h_to_4h.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.20.mlp.dense_h_to_4h.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.20.post_attention_layernorm.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.20.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.21.attention.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.21.attention.dense.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.21.attention.dense.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.21.attention.masked_bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.21.attention.query_key_value.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.21.attention.query_key_value.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.21.attention.rotary_emb.inv_freq": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.21.input_layernorm.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.21.input_layernorm.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.21.mlp.dense_4h_to_h.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.21.mlp.dense_4h_to_h.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.21.mlp.dense_h_to_4h.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.21.mlp.dense_h_to_4h.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.21.post_attention_layernorm.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.21.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.22.attention.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.22.attention.dense.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.22.attention.dense.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.22.attention.masked_bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.22.attention.query_key_value.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.22.attention.query_key_value.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.22.attention.rotary_emb.inv_freq": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.22.input_layernorm.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.22.input_layernorm.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.22.mlp.dense_4h_to_h.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.22.mlp.dense_4h_to_h.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.22.mlp.dense_h_to_4h.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.22.mlp.dense_h_to_4h.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.22.post_attention_layernorm.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.22.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.23.attention.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.23.attention.dense.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.23.attention.dense.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.23.attention.masked_bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.23.attention.query_key_value.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.23.attention.query_key_value.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.23.attention.rotary_emb.inv_freq": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.23.input_layernorm.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.23.input_layernorm.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.23.mlp.dense_4h_to_h.bias": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.23.mlp.dense_4h_to_h.weight": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.23.mlp.dense_h_to_4h.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.23.mlp.dense_h_to_4h.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.23.post_attention_layernorm.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.23.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.24.attention.bias": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.24.attention.dense.bias": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.24.attention.dense.weight": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.24.attention.masked_bias": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.24.attention.query_key_value.bias": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.24.attention.query_key_value.weight": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.24.attention.rotary_emb.inv_freq": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.24.input_layernorm.bias": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.24.input_layernorm.weight": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.24.mlp.dense_4h_to_h.bias": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.24.mlp.dense_4h_to_h.weight": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.24.mlp.dense_h_to_4h.bias": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.24.mlp.dense_h_to_4h.weight": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.24.post_attention_layernorm.bias": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.24.post_attention_layernorm.weight": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.25.attention.bias": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.25.attention.dense.bias": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.25.attention.dense.weight": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.25.attention.masked_bias": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.25.attention.query_key_value.bias": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.25.attention.query_key_value.weight": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.25.attention.rotary_emb.inv_freq": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.25.input_layernorm.bias": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.25.input_layernorm.weight": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.25.mlp.dense_4h_to_h.bias": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.25.mlp.dense_4h_to_h.weight": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.25.mlp.dense_h_to_4h.bias": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.25.mlp.dense_h_to_4h.weight": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.25.post_attention_layernorm.bias": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.25.post_attention_layernorm.weight": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.26.attention.bias": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.26.attention.dense.bias": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.26.attention.dense.weight": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.26.attention.masked_bias": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.26.attention.query_key_value.bias": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.26.attention.query_key_value.weight": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.26.attention.rotary_emb.inv_freq": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.26.input_layernorm.bias": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.26.input_layernorm.weight": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.26.mlp.dense_4h_to_h.bias": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.26.mlp.dense_4h_to_h.weight": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.26.mlp.dense_h_to_4h.bias": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.26.mlp.dense_h_to_4h.weight": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.26.post_attention_layernorm.bias": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.26.post_attention_layernorm.weight": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.27.attention.bias": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.27.attention.dense.bias": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.27.attention.dense.weight": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.27.attention.masked_bias": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.27.attention.query_key_value.bias": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.27.attention.query_key_value.weight": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.27.attention.rotary_emb.inv_freq": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.27.input_layernorm.bias": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.27.input_layernorm.weight": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.27.mlp.dense_4h_to_h.bias": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.27.mlp.dense_4h_to_h.weight": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.27.mlp.dense_h_to_4h.bias": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.27.mlp.dense_h_to_4h.weight": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.27.post_attention_layernorm.bias": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.27.post_attention_layernorm.weight": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.28.attention.bias": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.28.attention.dense.bias": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.28.attention.dense.weight": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.28.attention.masked_bias": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.28.attention.query_key_value.bias": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.28.attention.query_key_value.weight": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.28.attention.rotary_emb.inv_freq": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.28.input_layernorm.bias": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.28.input_layernorm.weight": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.28.mlp.dense_4h_to_h.bias": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.28.mlp.dense_4h_to_h.weight": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.28.mlp.dense_h_to_4h.bias": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.28.mlp.dense_h_to_4h.weight": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.28.post_attention_layernorm.bias": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.28.post_attention_layernorm.weight": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.29.attention.bias": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.29.attention.dense.bias": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.29.attention.dense.weight": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.29.attention.masked_bias": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.29.attention.query_key_value.bias": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.29.attention.query_key_value.weight": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.29.attention.rotary_emb.inv_freq": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.29.input_layernorm.bias": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.29.input_layernorm.weight": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.29.mlp.dense_4h_to_h.bias": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.29.mlp.dense_4h_to_h.weight": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.29.mlp.dense_h_to_4h.bias": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.29.mlp.dense_h_to_4h.weight": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.29.post_attention_layernorm.bias": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.29.post_attention_layernorm.weight": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.3.attention.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.3.attention.dense.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.3.attention.dense.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.3.attention.masked_bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.3.attention.query_key_value.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.3.attention.query_key_value.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.3.attention.rotary_emb.inv_freq": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.3.input_layernorm.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.3.input_layernorm.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.3.mlp.dense_4h_to_h.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.3.mlp.dense_4h_to_h.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.3.mlp.dense_h_to_4h.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.3.mlp.dense_h_to_4h.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.3.post_attention_layernorm.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.3.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.30.attention.bias": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.30.attention.dense.bias": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.30.attention.dense.weight": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.30.attention.masked_bias": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.30.attention.query_key_value.bias": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.30.attention.query_key_value.weight": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.30.attention.rotary_emb.inv_freq": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.30.input_layernorm.bias": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.30.input_layernorm.weight": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.30.mlp.dense_4h_to_h.bias": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.30.mlp.dense_4h_to_h.weight": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.30.mlp.dense_h_to_4h.bias": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.30.mlp.dense_h_to_4h.weight": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.30.post_attention_layernorm.bias": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.30.post_attention_layernorm.weight": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.31.attention.bias": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.31.attention.dense.bias": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.31.attention.dense.weight": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.31.attention.masked_bias": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.31.attention.query_key_value.bias": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.31.attention.query_key_value.weight": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.31.attention.rotary_emb.inv_freq": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.31.input_layernorm.bias": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.31.input_layernorm.weight": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.31.mlp.dense_4h_to_h.bias": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.31.mlp.dense_4h_to_h.weight": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.31.mlp.dense_h_to_4h.bias": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.31.mlp.dense_h_to_4h.weight": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.31.post_attention_layernorm.bias": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.31.post_attention_layernorm.weight": "pytorch_model-00002-of-00002.bin", + "gpt_neox.layers.4.attention.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.4.attention.dense.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.4.attention.dense.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.4.attention.masked_bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.4.attention.query_key_value.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.4.attention.query_key_value.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.4.attention.rotary_emb.inv_freq": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.4.input_layernorm.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.4.input_layernorm.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.4.mlp.dense_4h_to_h.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.4.mlp.dense_4h_to_h.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.4.mlp.dense_h_to_4h.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.4.mlp.dense_h_to_4h.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.4.post_attention_layernorm.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.4.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.5.attention.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.5.attention.dense.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.5.attention.dense.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.5.attention.masked_bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.5.attention.query_key_value.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.5.attention.query_key_value.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.5.attention.rotary_emb.inv_freq": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.5.input_layernorm.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.5.input_layernorm.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.5.mlp.dense_4h_to_h.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.5.mlp.dense_4h_to_h.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.5.mlp.dense_h_to_4h.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.5.mlp.dense_h_to_4h.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.5.post_attention_layernorm.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.5.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.6.attention.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.6.attention.dense.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.6.attention.dense.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.6.attention.masked_bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.6.attention.query_key_value.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.6.attention.query_key_value.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.6.attention.rotary_emb.inv_freq": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.6.input_layernorm.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.6.input_layernorm.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.6.mlp.dense_4h_to_h.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.6.mlp.dense_4h_to_h.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.6.mlp.dense_h_to_4h.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.6.mlp.dense_h_to_4h.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.6.post_attention_layernorm.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.6.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.7.attention.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.7.attention.dense.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.7.attention.dense.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.7.attention.masked_bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.7.attention.query_key_value.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.7.attention.query_key_value.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.7.attention.rotary_emb.inv_freq": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.7.input_layernorm.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.7.input_layernorm.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.7.mlp.dense_4h_to_h.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.7.mlp.dense_4h_to_h.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.7.mlp.dense_h_to_4h.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.7.mlp.dense_h_to_4h.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.7.post_attention_layernorm.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.7.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.8.attention.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.8.attention.dense.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.8.attention.dense.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.8.attention.masked_bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.8.attention.query_key_value.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.8.attention.query_key_value.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.8.attention.rotary_emb.inv_freq": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.8.input_layernorm.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.8.input_layernorm.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.8.mlp.dense_4h_to_h.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.8.mlp.dense_4h_to_h.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.8.mlp.dense_h_to_4h.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.8.mlp.dense_h_to_4h.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.8.post_attention_layernorm.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.8.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.9.attention.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.9.attention.dense.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.9.attention.dense.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.9.attention.masked_bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.9.attention.query_key_value.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.9.attention.query_key_value.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.9.attention.rotary_emb.inv_freq": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.9.input_layernorm.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.9.input_layernorm.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.9.mlp.dense_4h_to_h.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.9.mlp.dense_4h_to_h.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.9.mlp.dense_h_to_4h.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.9.mlp.dense_h_to_4h.weight": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.9.post_attention_layernorm.bias": "pytorch_model-00001-of-00002.bin", + "gpt_neox.layers.9.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin" + } +} diff --git a/special_tokens_map.json b/special_tokens_map.json new file mode 100644 index 0000000..0204ed1 --- /dev/null +++ b/special_tokens_map.json @@ -0,0 +1,5 @@ +{ + "bos_token": "<|endoftext|>", + "eos_token": "<|endoftext|>", + "unk_token": "<|endoftext|>" +} diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..79c8a4c --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3cf430678137c8491ca82fb7092ee49e44ad38857fffe1e4a4a5ed860139a5b8 +size 2113738 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..af28df4 --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,9 @@ +{ + "add_prefix_space": false, + "bos_token": "<|endoftext|>", + "clean_up_tokenization_spaces": true, + "eos_token": "<|endoftext|>", + "model_max_length": 2048, + "tokenizer_class": "GPTNeoXTokenizer", + "unk_token": "<|endoftext|>" +}