From a89ebbf245d16e086a28f5e013c13794306a7be3 Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Thu, 21 May 2026 09:12:12 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: EleutherAI/polyglot-ko-3.8b Source: Original Platform --- .gitattributes | 49 +++ README.md | 203 +++++++++++++ config.json | 26 ++ configuration.json | 1 + generation_config.json | 6 + model-00001-of-00008.safetensors | 3 + model-00002-of-00008.safetensors | 3 + model-00003-of-00008.safetensors | 3 + model-00004-of-00008.safetensors | 3 + model-00005-of-00008.safetensors | 3 + model-00006-of-00008.safetensors | 3 + model-00007-of-00008.safetensors | 3 + model-00008-of-00008.safetensors | 3 + model.safetensors.index.json | 491 +++++++++++++++++++++++++++++++ pytorch_model.bin | 3 + special_tokens_map.json | 11 + tokenizer.json | 3 + tokenizer_config.json | 6 + 18 files changed, 823 insertions(+) create mode 100644 .gitattributes create mode 100644 README.md create mode 100644 config.json create mode 100644 configuration.json create mode 100644 generation_config.json create mode 100644 model-00001-of-00008.safetensors create mode 100644 model-00002-of-00008.safetensors create mode 100644 model-00003-of-00008.safetensors create mode 100644 model-00004-of-00008.safetensors create mode 100644 model-00005-of-00008.safetensors create mode 100644 model-00006-of-00008.safetensors create mode 100644 model-00007-of-00008.safetensors create mode 100644 model-00008-of-00008.safetensors create mode 100644 model.safetensors.index.json create mode 100644 pytorch_model.bin create mode 100644 special_tokens_map.json create mode 100644 tokenizer.json create mode 100644 tokenizer_config.json diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..21b3632 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,49 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bin.* filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zstandard filter=lfs diff=lfs merge=lfs -text +*.tfevents* filter=lfs diff=lfs merge=lfs -text +*.db* filter=lfs diff=lfs merge=lfs -text +*.ark* filter=lfs diff=lfs merge=lfs -text +**/*ckpt*data* filter=lfs diff=lfs merge=lfs -text +**/*ckpt*.meta filter=lfs diff=lfs merge=lfs -text +**/*ckpt*.index filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.gguf* filter=lfs diff=lfs merge=lfs -text +*.ggml filter=lfs diff=lfs merge=lfs -text +*.llamafile* filter=lfs diff=lfs merge=lfs -text +*.pt2 filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text + +tokenizer.json filter=lfs diff=lfs merge=lfs -text \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..425c962 --- /dev/null +++ b/README.md @@ -0,0 +1,203 @@ +--- +language: +- ko +tags: +- pytorch +- causal-lm +license: apache-2.0 + +--- +# Polyglot-Ko-3.8B + +## Model Description +Polyglot-Ko is a series of large-scale Korean autoregressive language models made by the EleutherAI polyglot team. + +| Hyperparameter | Value | +|----------------------|----------------------------------------------------------------------------------------------------------------------------------------| +| \\(n_{parameters}\\) | 3,809,974,272 | +| \\(n_{layers}\\) | 32 | +| \\(d_{model}\\) | 3,072 | +| \\(d_{ff}\\) | 12,288 | +| \\(n_{heads}\\) | 24 | +| \\(d_{head}\\) | 128 | +| \\(n_{ctx}\\) | 2,048 | +| \\(n_{vocab}\\) | 30,003 / 30,080 | +| Positional Encoding | [Rotary Position Embedding (RoPE)](https://arxiv.org/abs/2104.09864) | +| RoPE Dimensions | [64](https://github.com/kingoflolz/mesh-transformer-jax/blob/f2aa66e0925de6593dcbb70e72399b97b4130482/mesh_transformer/layers.py#L223) | + +The model consists of 32 transformer layers with a model dimension of 3072, and a feedforward dimension of 12288. The model +dimension is split into 24 heads, each with a dimension of 128. Rotary Position Embedding (RoPE) is applied to 64 +dimensions of each head. The model is trained with a tokenization vocabulary of 30003. + +## Training data + +Polyglot-Ko-3.8B was trained on 863 GB of Korean language data (1.2TB before processing), a large-scale dataset curated by [TUNiB](https://tunib.ai/). The data collection process has abided by South Korean laws. This dataset was collected for the purpose of training Polyglot-Ko models, so it will not be released for public use. + +| Source |Size (GB) | Link | +|-------------------------------------|---------|------------------------------------------| +| Korean blog posts | 682.3 | - | +| Korean news dataset | 87.0 | - | +| Modu corpus | 26.4 |corpus.korean.go.kr | +| Korean patent dataset | 19.0 | - | +| Korean Q & A dataset | 18.1 | - | +| KcBert dataset | 12.7 | github.com/Beomi/KcBERT | +| Korean fiction dataset | 6.1 | - | +| Korean online comments | 4.2 | - | +| Korean wikipedia | 1.4 | ko.wikipedia.org | +| Clova call | < 1.0 | github.com/clovaai/ClovaCall | +| Naver sentiment movie corpus | < 1.0 | github.com/e9t/nsmc | +| Korean hate speech dataset | < 1.0 | - | +| Open subtitles | < 1.0 | opus.nlpl.eu/OpenSubtitles.php | +| AIHub various tasks datasets | < 1.0 |aihub.or.kr | +| Standard Korean language dictionary | < 1.0 | stdict.korean.go.kr/main/main.do | + +Furthermore, in order to avoid the model memorizing and generating personally identifiable information (PII) in the training data, we masked out the following sensitive information in the pre-processing stage: + +* `<|acc|>` : bank account number +* `<|rrn|>` : resident registration number +* `<|tell|>` : phone number + +## Training procedure +Polyglot-Ko-3.8B was trained for 219 billion tokens over 105,000 steps on 256 A100 GPUs with the [GPT-NeoX framework](https://github.com/EleutherAI/gpt-neox). It was trained as an autoregressive language model, using cross-entropy loss to maximize the likelihood of predicting the next token. + +## How to use + +This model can be easily loaded using the `AutoModelForCausalLM` class: + +```python +from transformers import AutoTokenizer, AutoModelForCausalLM + +tokenizer = AutoTokenizer.from_pretrained("EleutherAI/polyglot-ko-3.8b") +model = AutoModelForCausalLM.from_pretrained("EleutherAI/polyglot-ko-3.8b") +``` + +## Evaluation results + +We evaluate Polyglot-Ko-3.8B on [KOBEST dataset](https://arxiv.org/abs/2204.04541), a benchmark with 5 downstream tasks, against comparable models such as skt/ko-gpt-trinity-1.2B-v0.5, kakaobrain/kogpt and facebook/xglm-7.5B, using the prompts provided in the paper. + +The following tables show the results when the number of few-shot examples differ. You can reproduce these results using the [polyglot branch of lm-evaluation-harness](https://github.com/EleutherAI/lm-evaluation-harness/tree/polyglot) and the following scripts. For a fair comparison, all models were run under the same conditions and using the same prompts. In the tables, `n` refers to the number of few-shot examples. + +In case of WiC dataset, all models show random performance. + +```console +python main.py \ + --model gpt2 \ + --model_args pretrained='EleutherAI/polyglot-ko-3.8b' \ + --tasks kobest_copa,kobest_hellaswag \ + --num_fewshot $YOUR_NUM_FEWSHOT \ + --batch_size $YOUR_BATCH_SIZE \ + --device $YOUR_DEVICE \ + --output_path $/path/to/output/ +``` + +### COPA (F1) + +| Model | params | 0-shot | 5-shot | 10-shot | 50-shot | +|----------------------------------------------------------------------------------------------|--------|--------|--------|---------|---------| +| [skt/ko-gpt-trinity-1.2B-v0.5](https://huggingface.co/skt/ko-gpt-trinity-1.2B-v0.5) | 1.2B | 0.6696 | 0.6477 | 0.6419 | 0.6514 | +| [kakaobrain/kogpt](https://huggingface.co/kakaobrain/kogpt) | 6.0B | 0.7345 | 0.7287 | 0.7277 | 0.7479 | +| [facebook/xglm-7.5B](https://huggingface.co/facebook/xglm-7.5B) | 7.5B | 0.6723 | 0.6731 | 0.6769 | 0.7119 | +| [EleutherAI/polyglot-ko-1.3b](https://huggingface.co/EleutherAI/polyglot-ko-1.3b) | 1.3B | 0.7196 | 0.7193 | 0.7204 | 0.7206 | +| **[EleutherAI/polyglot-ko-3.8b](https://huggingface.co/EleutherAI/polyglot-ko-3.8b) (this)** | **3.8B** | **0.7595** | **0.7608** | **0.7638** | **0.7788** | +| [EleutherAI/polyglot-ko-5.8b](https://huggingface.co/EleutherAI/polyglot-ko-5.8b) | 5.8B | 0.7745 | 0.7676 | 0.7775 | 0.7887 | +| [EleutherAI/polyglot-ko-12.8b](https://huggingface.co/EleutherAI/polyglot-ko-12.8b) | 12.8B | 0.7937 | 0.8108 | 0.8037 | 0.8369 | + + + +### HellaSwag (F1) + +| Model | params | 0-shot | 5-shot | 10-shot | 50-shot | +|----------------------------------------------------------------------------------------------|--------|--------|--------|---------|---------| +| [skt/ko-gpt-trinity-1.2B-v0.5](https://huggingface.co/skt/ko-gpt-trinity-1.2B-v0.5) | 1.2B | 0.5243 | 0.5272 | 0.5166 | 0.5352 | +| [kakaobrain/kogpt](https://huggingface.co/kakaobrain/kogpt) | 6.0B | 0.5590 | 0.5833 | 0.5828 | 0.5907 | +| [facebook/xglm-7.5B](https://huggingface.co/facebook/xglm-7.5B) | 7.5B | 0.5665 | 0.5689 | 0.5565 | 0.5622 | +| [EleutherAI/polyglot-ko-1.3b](https://huggingface.co/EleutherAI/polyglot-ko-1.3b) | 1.3B | 0.5247 | 0.5260 | 0.5278 | 0.5427 | +| **[EleutherAI/polyglot-ko-3.8b](https://huggingface.co/EleutherAI/polyglot-ko-3.8b) (this)** | **3.8B** | **0.5707** | **0.5830** | **0.5670** | **0.5787** | +| [EleutherAI/polyglot-ko-5.8b](https://huggingface.co/EleutherAI/polyglot-ko-5.8b) | 5.8B | 0.5976 | 0.5998 | 0.5979 | 0.6208 | +| [EleutherAI/polyglot-ko-12.8b](https://huggingface.co/EleutherAI/polyglot-ko-12.8b) | 12.8B | 0.5954 | 0.6306 | 0.6098 | 0.6118 | + + + +### BoolQ (F1) + +| Model | params | 0-shot | 5-shot | 10-shot | 50-shot | +|----------------------------------------------------------------------------------------------|--------|--------|--------|---------|---------| +| [skt/ko-gpt-trinity-1.2B-v0.5](https://huggingface.co/skt/ko-gpt-trinity-1.2B-v0.5) | 1.2B | 0.3356 | 0.4014 | 0.3640 | 0.3560 | +| [kakaobrain/kogpt](https://huggingface.co/kakaobrain/kogpt) | 6.0B | 0.4514 | 0.5981 | 0.5499 | 0.5202 | +| [facebook/xglm-7.5B](https://huggingface.co/facebook/xglm-7.5B) | 7.5B | 0.4464 | 0.3324 | 0.3324 | 0.3324 | +| [EleutherAI/polyglot-ko-1.3b](https://huggingface.co/EleutherAI/polyglot-ko-1.3b) | 1.3B | 0.3552 | 0.4751 | 0.4109 | 0.4038 | +| **[EleutherAI/polyglot-ko-3.8b](https://huggingface.co/EleutherAI/polyglot-ko-3.8b) (this)** | **3.8B** | **0.4320** | **0.5263** | **0.4930** | **0.4038** | +| [EleutherAI/polyglot-ko-5.8b](https://huggingface.co/EleutherAI/polyglot-ko-5.8b) | 5.8B | 0.4356 | 0.5698 | 0.5187 | 0.5236 | +| [EleutherAI/polyglot-ko-12.8b](https://huggingface.co/EleutherAI/polyglot-ko-12.8b) | 12.8B | 0.4818 | 0.6041 | 0.6289 | 0.6448 | + + + + +### SentiNeg (F1) + +| Model | params | 0-shot | 5-shot | 10-shot | 50-shot | +|----------------------------------------------------------------------------------------------|--------|--------|--------|---------|---------| +| [skt/ko-gpt-trinity-1.2B-v0.5](https://huggingface.co/skt/ko-gpt-trinity-1.2B-v0.5) | 1.2B | 0.6065 | 0.6878 | 0.7280 | 0.8413 | +| [kakaobrain/kogpt](https://huggingface.co/kakaobrain/kogpt) | 6.0B | 0.3747 | 0.8942 | 0.9294 | 0.9698 | +| [facebook/xglm-7.5B](https://huggingface.co/facebook/xglm-7.5B) | 7.5B | 0.3578 | 0.4471 | 0.3964 | 0.5271 | +| [EleutherAI/polyglot-ko-1.3b](https://huggingface.co/EleutherAI/polyglot-ko-1.3b) | 1.3B | 0.6790 | 0.6257 | 0.5514 | 0.7851 | +| **[EleutherAI/polyglot-ko-3.8b](https://huggingface.co/EleutherAI/polyglot-ko-3.8b) (this)** | **3.8B** | **0.4858** | **0.7950** | **0.7320** | **0.7851** | +| [EleutherAI/polyglot-ko-5.8b](https://huggingface.co/EleutherAI/polyglot-ko-5.8b) | 5.8B | 0.3394 | 0.8841 | 0.8808 | 0.9521 | +| [EleutherAI/polyglot-ko-12.8b](https://huggingface.co/EleutherAI/polyglot-ko-12.8b) | 12.8B | 0.9117 | 0.9015 | 0.9345 | 0.9723 | + + + + +### WiC (F1) + +| Model | params | 0-shot | 5-shot | 10-shot | 50-shot | +|----------------------------------------------------------------------------------------------|--------|--------|--------|---------|---------| +| [skt/ko-gpt-trinity-1.2B-v0.5](https://huggingface.co/skt/ko-gpt-trinity-1.2B-v0.5) | 1.2B | 0.3290 | 0.4313 | 0.4001 | 0.3621 | +| [kakaobrain/kogpt](https://huggingface.co/kakaobrain/kogpt) | 6.0B | 0.3526 | 0.4775 | 0.4358 | 0.4061 | +| [facebook/xglm-7.5B](https://huggingface.co/facebook/xglm-7.5B) | 7.5B | 0.3280 | 0.4903 | 0.4945 | 0.3656 | +| [EleutherAI/polyglot-ko-1.3b](https://huggingface.co/EleutherAI/polyglot-ko-1.3b) | 1.3B | 0.3297 | 0.4850 | 0.4650 | 0.3290 | +| **[EleutherAI/polyglot-ko-3.8b](https://huggingface.co/EleutherAI/polyglot-ko-3.8b) (this)** | **3.8B** | **0.3390** | **0.4944** | **0.4203** | **0.3835** | +| [EleutherAI/polyglot-ko-5.8b](https://huggingface.co/EleutherAI/polyglot-ko-5.8b) | 5.8B | 0.3913 | 0.4688 | 0.4189 | 0.3910 | +| [EleutherAI/polyglot-ko-12.8b](https://huggingface.co/EleutherAI/polyglot-ko-12.8b) | 12.8B | 0.3985 | 0.3683 | 0.3307 | 0.3273 | + + + + +## Limitations and Biases + +Polyglot-Ko has been trained to optimize next token prediction. Language models such as this are often used for a wide variety of tasks and it is important to be aware of possible unexpected outcomes. For instance, Polyglot-Ko will not always return the most factual or accurate response but the most statistically likely one. In addition, Polyglot may produce socially unacceptable or offensive content. We recommend having a human curator or other filtering mechanism to censor sensitive content. + +## Citation and Related Information +### BibTeX entry +If you find our work useful, please consider citing: +```bibtex +@misc{ko2023technical, + title={A Technical Report for Polyglot-Ko: Open-Source Large-Scale Korean Language Models}, + author={Hyunwoong Ko and Kichang Yang and Minho Ryu and Taekyoon Choi and Seungmu Yang and jiwung Hyun and Sungho Park}, + year={2023}, + eprint={2306.02254}, + archivePrefix={arXiv}, + primaryClass={cs.CL} +} +``` + +### Licensing +All our models are licensed under the terms of the Apache License 2.0. + +``` +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +``` + +### Acknowledgement + +This project was made possible thanks to the computing resources from [Stability.ai](https://stability.ai), and thanks to [TUNiB](https://tunib.ai) for providing a large-scale Korean dataset for this work. diff --git a/config.json b/config.json new file mode 100644 index 0000000..25966d0 --- /dev/null +++ b/config.json @@ -0,0 +1,26 @@ +{ + "_name_or_path": "./polyglot-ko-3.8b/", + "architectures": [ + "GPTNeoXForCausalLM" + ], + "bos_token_id": 0, + "classifier_dropout": 0.1, + "eos_token_id": 2, + "hidden_act": "gelu", + "hidden_size": 3072, + "initializer_range": 0.02, + "intermediate_size": 12288, + "layer_norm_eps": 1e-05, + "max_position_embeddings": 2048, + "model_type": "gpt_neox", + "num_attention_heads": 24, + "num_hidden_layers": 32, + "rotary_emb_base": 10000, + "rotary_pct": 0.5, + "tie_word_embeddings": false, + "torch_dtype": "float16", + "transformers_version": "4.29.2", + "use_cache": true, + "use_parallel_residual": true, + "vocab_size": 30080 +} diff --git a/configuration.json b/configuration.json new file mode 100644 index 0000000..bbeeda1 --- /dev/null +++ b/configuration.json @@ -0,0 +1 @@ +{"framework": "pytorch", "task": "text-generation", "allow_remote": true} \ No newline at end of file diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..7a838ce --- /dev/null +++ b/generation_config.json @@ -0,0 +1,6 @@ +{ + "_from_model_config": true, + "bos_token_id": 0, + "eos_token_id": 2, + "transformers_version": "4.29.2" +} diff --git a/model-00001-of-00008.safetensors b/model-00001-of-00008.safetensors new file mode 100644 index 0000000..0158c99 --- /dev/null +++ b/model-00001-of-00008.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eb686e62d8f44c95de8a6cfe1a0c70fd21a80cf8853ad1557a7655e792548691 +size 956859264 diff --git a/model-00002-of-00008.safetensors b/model-00002-of-00008.safetensors new file mode 100644 index 0000000..6ef69c8 --- /dev/null +++ b/model-00002-of-00008.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d906f3c3cfd9bffcb05cc95bbb9daf7ceb3cbeed08e9de4ce46c01230d73e38f +size 998596040 diff --git a/model-00003-of-00008.safetensors b/model-00003-of-00008.safetensors new file mode 100644 index 0000000..d644a0a --- /dev/null +++ b/model-00003-of-00008.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0e4f72d41324f36aea931311bceaf685fbde30ceb5de9eb03ad43c6d0cbf20ad +size 1002797426 diff --git a/model-00004-of-00008.safetensors b/model-00004-of-00008.safetensors new file mode 100644 index 0000000..6d00652 --- /dev/null +++ b/model-00004-of-00008.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dcaf868ec73c80c57644946344e1be703e9cde961f10eb7f6db89b5ae19ccbde +size 998596336 diff --git a/model-00005-of-00008.safetensors b/model-00005-of-00008.safetensors new file mode 100644 index 0000000..69da8c6 --- /dev/null +++ b/model-00005-of-00008.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e140c307650709d0e070c6afbc5a6eca734a114eb436ecd4381f072efcf271c3 +size 998596096 diff --git a/model-00006-of-00008.safetensors b/model-00006-of-00008.safetensors new file mode 100644 index 0000000..4057792 --- /dev/null +++ b/model-00006-of-00008.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7ba948db6174d75df12ae550692cbedbdb990939247b6ea3a3cabd4e1661fc5a +size 1002797458 diff --git a/model-00007-of-00008.safetensors b/model-00007-of-00008.safetensors new file mode 100644 index 0000000..13348fa --- /dev/null +++ b/model-00007-of-00008.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:89054f5b0f9af5abd3aff9857cdc1365d5dc3f77b89f965c956325e663a1d389 +size 998596336 diff --git a/model-00008-of-00008.safetensors b/model-00008-of-00008.safetensors new file mode 100644 index 0000000..4cd4014 --- /dev/null +++ b/model-00008-of-00008.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:efcafff4d10a4dec766a6e20d5830ac907b42dccab7cf9c60c3ad7aeb30de849 +size 797387164 diff --git a/model.safetensors.index.json b/model.safetensors.index.json new file mode 100644 index 0000000..c090748 --- /dev/null +++ b/model.safetensors.index.json @@ -0,0 +1,491 @@ +{ + "metadata": { + "total_size": 7636729920.0 + }, + "weight_map": { + "embed_out.weight": "model-00008-of-00008.safetensors", + "gpt_neox.embed_in.weight": "model-00001-of-00008.safetensors", + "gpt_neox.final_layer_norm.bias": "model-00008-of-00008.safetensors", + "gpt_neox.final_layer_norm.weight": "model-00008-of-00008.safetensors", + "gpt_neox.layers.0.attention.bias": "model-00001-of-00008.safetensors", + "gpt_neox.layers.0.attention.dense.bias": "model-00001-of-00008.safetensors", + "gpt_neox.layers.0.attention.dense.weight": "model-00001-of-00008.safetensors", + "gpt_neox.layers.0.attention.masked_bias": "model-00001-of-00008.safetensors", + "gpt_neox.layers.0.attention.query_key_value.bias": "model-00001-of-00008.safetensors", + "gpt_neox.layers.0.attention.query_key_value.weight": "model-00001-of-00008.safetensors", + "gpt_neox.layers.0.attention.rotary_emb.inv_freq": "model-00001-of-00008.safetensors", + "gpt_neox.layers.0.input_layernorm.bias": "model-00001-of-00008.safetensors", + "gpt_neox.layers.0.input_layernorm.weight": "model-00001-of-00008.safetensors", + "gpt_neox.layers.0.mlp.dense_4h_to_h.bias": "model-00001-of-00008.safetensors", + "gpt_neox.layers.0.mlp.dense_4h_to_h.weight": "model-00001-of-00008.safetensors", + "gpt_neox.layers.0.mlp.dense_h_to_4h.bias": "model-00001-of-00008.safetensors", + "gpt_neox.layers.0.mlp.dense_h_to_4h.weight": "model-00001-of-00008.safetensors", + "gpt_neox.layers.0.post_attention_layernorm.bias": "model-00001-of-00008.safetensors", + "gpt_neox.layers.0.post_attention_layernorm.weight": "model-00001-of-00008.safetensors", + "gpt_neox.layers.1.attention.bias": "model-00001-of-00008.safetensors", + "gpt_neox.layers.1.attention.dense.bias": "model-00001-of-00008.safetensors", + "gpt_neox.layers.1.attention.dense.weight": "model-00001-of-00008.safetensors", + "gpt_neox.layers.1.attention.masked_bias": "model-00001-of-00008.safetensors", + "gpt_neox.layers.1.attention.query_key_value.bias": "model-00001-of-00008.safetensors", + "gpt_neox.layers.1.attention.query_key_value.weight": "model-00001-of-00008.safetensors", + "gpt_neox.layers.1.attention.rotary_emb.inv_freq": "model-00001-of-00008.safetensors", + "gpt_neox.layers.1.input_layernorm.bias": "model-00001-of-00008.safetensors", + "gpt_neox.layers.1.input_layernorm.weight": "model-00001-of-00008.safetensors", + "gpt_neox.layers.1.mlp.dense_4h_to_h.bias": "model-00001-of-00008.safetensors", + "gpt_neox.layers.1.mlp.dense_4h_to_h.weight": "model-00001-of-00008.safetensors", + "gpt_neox.layers.1.mlp.dense_h_to_4h.bias": "model-00001-of-00008.safetensors", + "gpt_neox.layers.1.mlp.dense_h_to_4h.weight": "model-00001-of-00008.safetensors", + "gpt_neox.layers.1.post_attention_layernorm.bias": "model-00001-of-00008.safetensors", + "gpt_neox.layers.1.post_attention_layernorm.weight": "model-00001-of-00008.safetensors", + "gpt_neox.layers.10.attention.bias": "model-00003-of-00008.safetensors", + "gpt_neox.layers.10.attention.dense.bias": "model-00003-of-00008.safetensors", + "gpt_neox.layers.10.attention.dense.weight": "model-00003-of-00008.safetensors", + "gpt_neox.layers.10.attention.masked_bias": "model-00003-of-00008.safetensors", + "gpt_neox.layers.10.attention.query_key_value.bias": "model-00003-of-00008.safetensors", + "gpt_neox.layers.10.attention.query_key_value.weight": "model-00003-of-00008.safetensors", + "gpt_neox.layers.10.attention.rotary_emb.inv_freq": "model-00003-of-00008.safetensors", + "gpt_neox.layers.10.input_layernorm.bias": "model-00003-of-00008.safetensors", + "gpt_neox.layers.10.input_layernorm.weight": "model-00003-of-00008.safetensors", + "gpt_neox.layers.10.mlp.dense_4h_to_h.bias": "model-00003-of-00008.safetensors", + "gpt_neox.layers.10.mlp.dense_4h_to_h.weight": "model-00003-of-00008.safetensors", + "gpt_neox.layers.10.mlp.dense_h_to_4h.bias": "model-00003-of-00008.safetensors", + "gpt_neox.layers.10.mlp.dense_h_to_4h.weight": "model-00003-of-00008.safetensors", + "gpt_neox.layers.10.post_attention_layernorm.bias": "model-00003-of-00008.safetensors", + "gpt_neox.layers.10.post_attention_layernorm.weight": "model-00003-of-00008.safetensors", + "gpt_neox.layers.11.attention.bias": "model-00003-of-00008.safetensors", + "gpt_neox.layers.11.attention.dense.bias": "model-00003-of-00008.safetensors", + "gpt_neox.layers.11.attention.dense.weight": "model-00003-of-00008.safetensors", + "gpt_neox.layers.11.attention.masked_bias": "model-00003-of-00008.safetensors", + "gpt_neox.layers.11.attention.query_key_value.bias": "model-00003-of-00008.safetensors", + "gpt_neox.layers.11.attention.query_key_value.weight": "model-00003-of-00008.safetensors", + "gpt_neox.layers.11.attention.rotary_emb.inv_freq": "model-00003-of-00008.safetensors", + "gpt_neox.layers.11.input_layernorm.bias": "model-00003-of-00008.safetensors", + "gpt_neox.layers.11.input_layernorm.weight": "model-00003-of-00008.safetensors", + "gpt_neox.layers.11.mlp.dense_4h_to_h.bias": "model-00003-of-00008.safetensors", + "gpt_neox.layers.11.mlp.dense_4h_to_h.weight": "model-00003-of-00008.safetensors", + "gpt_neox.layers.11.mlp.dense_h_to_4h.bias": "model-00003-of-00008.safetensors", + "gpt_neox.layers.11.mlp.dense_h_to_4h.weight": "model-00003-of-00008.safetensors", + "gpt_neox.layers.11.post_attention_layernorm.bias": "model-00003-of-00008.safetensors", + "gpt_neox.layers.11.post_attention_layernorm.weight": "model-00003-of-00008.safetensors", + "gpt_neox.layers.12.attention.bias": "model-00003-of-00008.safetensors", + "gpt_neox.layers.12.attention.dense.bias": "model-00004-of-00008.safetensors", + "gpt_neox.layers.12.attention.dense.weight": "model-00004-of-00008.safetensors", + "gpt_neox.layers.12.attention.masked_bias": "model-00003-of-00008.safetensors", + "gpt_neox.layers.12.attention.query_key_value.bias": "model-00004-of-00008.safetensors", + "gpt_neox.layers.12.attention.query_key_value.weight": "model-00004-of-00008.safetensors", + "gpt_neox.layers.12.attention.rotary_emb.inv_freq": "model-00003-of-00008.safetensors", + "gpt_neox.layers.12.input_layernorm.bias": "model-00003-of-00008.safetensors", + "gpt_neox.layers.12.input_layernorm.weight": "model-00003-of-00008.safetensors", + "gpt_neox.layers.12.mlp.dense_4h_to_h.bias": "model-00004-of-00008.safetensors", + "gpt_neox.layers.12.mlp.dense_4h_to_h.weight": "model-00004-of-00008.safetensors", + "gpt_neox.layers.12.mlp.dense_h_to_4h.bias": "model-00004-of-00008.safetensors", + "gpt_neox.layers.12.mlp.dense_h_to_4h.weight": "model-00004-of-00008.safetensors", + "gpt_neox.layers.12.post_attention_layernorm.bias": "model-00003-of-00008.safetensors", + "gpt_neox.layers.12.post_attention_layernorm.weight": "model-00003-of-00008.safetensors", + "gpt_neox.layers.13.attention.bias": "model-00004-of-00008.safetensors", + "gpt_neox.layers.13.attention.dense.bias": "model-00004-of-00008.safetensors", + "gpt_neox.layers.13.attention.dense.weight": "model-00004-of-00008.safetensors", + "gpt_neox.layers.13.attention.masked_bias": "model-00004-of-00008.safetensors", + "gpt_neox.layers.13.attention.query_key_value.bias": "model-00004-of-00008.safetensors", + "gpt_neox.layers.13.attention.query_key_value.weight": "model-00004-of-00008.safetensors", + "gpt_neox.layers.13.attention.rotary_emb.inv_freq": "model-00004-of-00008.safetensors", + "gpt_neox.layers.13.input_layernorm.bias": "model-00004-of-00008.safetensors", + "gpt_neox.layers.13.input_layernorm.weight": "model-00004-of-00008.safetensors", + "gpt_neox.layers.13.mlp.dense_4h_to_h.bias": "model-00004-of-00008.safetensors", + "gpt_neox.layers.13.mlp.dense_4h_to_h.weight": "model-00004-of-00008.safetensors", + "gpt_neox.layers.13.mlp.dense_h_to_4h.bias": "model-00004-of-00008.safetensors", + "gpt_neox.layers.13.mlp.dense_h_to_4h.weight": "model-00004-of-00008.safetensors", + "gpt_neox.layers.13.post_attention_layernorm.bias": "model-00004-of-00008.safetensors", + "gpt_neox.layers.13.post_attention_layernorm.weight": "model-00004-of-00008.safetensors", + "gpt_neox.layers.14.attention.bias": "model-00004-of-00008.safetensors", + "gpt_neox.layers.14.attention.dense.bias": "model-00004-of-00008.safetensors", + "gpt_neox.layers.14.attention.dense.weight": "model-00004-of-00008.safetensors", + "gpt_neox.layers.14.attention.masked_bias": "model-00004-of-00008.safetensors", + "gpt_neox.layers.14.attention.query_key_value.bias": "model-00004-of-00008.safetensors", + "gpt_neox.layers.14.attention.query_key_value.weight": "model-00004-of-00008.safetensors", + "gpt_neox.layers.14.attention.rotary_emb.inv_freq": "model-00004-of-00008.safetensors", + "gpt_neox.layers.14.input_layernorm.bias": "model-00004-of-00008.safetensors", + "gpt_neox.layers.14.input_layernorm.weight": "model-00004-of-00008.safetensors", + "gpt_neox.layers.14.mlp.dense_4h_to_h.bias": "model-00004-of-00008.safetensors", + "gpt_neox.layers.14.mlp.dense_4h_to_h.weight": "model-00004-of-00008.safetensors", + "gpt_neox.layers.14.mlp.dense_h_to_4h.bias": "model-00004-of-00008.safetensors", + "gpt_neox.layers.14.mlp.dense_h_to_4h.weight": "model-00004-of-00008.safetensors", + "gpt_neox.layers.14.post_attention_layernorm.bias": "model-00004-of-00008.safetensors", + "gpt_neox.layers.14.post_attention_layernorm.weight": "model-00004-of-00008.safetensors", + "gpt_neox.layers.15.attention.bias": "model-00004-of-00008.safetensors", + "gpt_neox.layers.15.attention.dense.bias": "model-00004-of-00008.safetensors", + "gpt_neox.layers.15.attention.dense.weight": "model-00004-of-00008.safetensors", + "gpt_neox.layers.15.attention.masked_bias": "model-00004-of-00008.safetensors", + "gpt_neox.layers.15.attention.query_key_value.bias": "model-00004-of-00008.safetensors", + "gpt_neox.layers.15.attention.query_key_value.weight": "model-00004-of-00008.safetensors", + "gpt_neox.layers.15.attention.rotary_emb.inv_freq": "model-00004-of-00008.safetensors", + "gpt_neox.layers.15.input_layernorm.bias": "model-00004-of-00008.safetensors", + "gpt_neox.layers.15.input_layernorm.weight": "model-00004-of-00008.safetensors", + "gpt_neox.layers.15.mlp.dense_4h_to_h.bias": "model-00004-of-00008.safetensors", + "gpt_neox.layers.15.mlp.dense_4h_to_h.weight": "model-00004-of-00008.safetensors", + "gpt_neox.layers.15.mlp.dense_h_to_4h.bias": "model-00004-of-00008.safetensors", + "gpt_neox.layers.15.mlp.dense_h_to_4h.weight": "model-00004-of-00008.safetensors", + "gpt_neox.layers.15.post_attention_layernorm.bias": "model-00004-of-00008.safetensors", + "gpt_neox.layers.15.post_attention_layernorm.weight": "model-00004-of-00008.safetensors", + "gpt_neox.layers.16.attention.bias": "model-00004-of-00008.safetensors", + "gpt_neox.layers.16.attention.dense.bias": "model-00004-of-00008.safetensors", + "gpt_neox.layers.16.attention.dense.weight": "model-00004-of-00008.safetensors", + "gpt_neox.layers.16.attention.masked_bias": "model-00004-of-00008.safetensors", + "gpt_neox.layers.16.attention.query_key_value.bias": "model-00004-of-00008.safetensors", + "gpt_neox.layers.16.attention.query_key_value.weight": "model-00004-of-00008.safetensors", + "gpt_neox.layers.16.attention.rotary_emb.inv_freq": "model-00004-of-00008.safetensors", + "gpt_neox.layers.16.input_layernorm.bias": "model-00004-of-00008.safetensors", + "gpt_neox.layers.16.input_layernorm.weight": "model-00004-of-00008.safetensors", + "gpt_neox.layers.16.mlp.dense_4h_to_h.bias": "model-00005-of-00008.safetensors", + "gpt_neox.layers.16.mlp.dense_4h_to_h.weight": "model-00005-of-00008.safetensors", + "gpt_neox.layers.16.mlp.dense_h_to_4h.bias": "model-00005-of-00008.safetensors", + "gpt_neox.layers.16.mlp.dense_h_to_4h.weight": "model-00005-of-00008.safetensors", + "gpt_neox.layers.16.post_attention_layernorm.bias": "model-00004-of-00008.safetensors", + "gpt_neox.layers.16.post_attention_layernorm.weight": "model-00004-of-00008.safetensors", + "gpt_neox.layers.17.attention.bias": "model-00005-of-00008.safetensors", + "gpt_neox.layers.17.attention.dense.bias": "model-00005-of-00008.safetensors", + "gpt_neox.layers.17.attention.dense.weight": "model-00005-of-00008.safetensors", + "gpt_neox.layers.17.attention.masked_bias": "model-00005-of-00008.safetensors", + "gpt_neox.layers.17.attention.query_key_value.bias": "model-00005-of-00008.safetensors", + "gpt_neox.layers.17.attention.query_key_value.weight": "model-00005-of-00008.safetensors", + "gpt_neox.layers.17.attention.rotary_emb.inv_freq": "model-00005-of-00008.safetensors", + "gpt_neox.layers.17.input_layernorm.bias": "model-00005-of-00008.safetensors", + "gpt_neox.layers.17.input_layernorm.weight": "model-00005-of-00008.safetensors", + "gpt_neox.layers.17.mlp.dense_4h_to_h.bias": "model-00005-of-00008.safetensors", + "gpt_neox.layers.17.mlp.dense_4h_to_h.weight": "model-00005-of-00008.safetensors", + "gpt_neox.layers.17.mlp.dense_h_to_4h.bias": "model-00005-of-00008.safetensors", + "gpt_neox.layers.17.mlp.dense_h_to_4h.weight": "model-00005-of-00008.safetensors", + "gpt_neox.layers.17.post_attention_layernorm.bias": "model-00005-of-00008.safetensors", + "gpt_neox.layers.17.post_attention_layernorm.weight": "model-00005-of-00008.safetensors", + "gpt_neox.layers.18.attention.bias": "model-00005-of-00008.safetensors", + "gpt_neox.layers.18.attention.dense.bias": "model-00005-of-00008.safetensors", + "gpt_neox.layers.18.attention.dense.weight": "model-00005-of-00008.safetensors", + "gpt_neox.layers.18.attention.masked_bias": "model-00005-of-00008.safetensors", + "gpt_neox.layers.18.attention.query_key_value.bias": "model-00005-of-00008.safetensors", + "gpt_neox.layers.18.attention.query_key_value.weight": "model-00005-of-00008.safetensors", + "gpt_neox.layers.18.attention.rotary_emb.inv_freq": "model-00005-of-00008.safetensors", + "gpt_neox.layers.18.input_layernorm.bias": "model-00005-of-00008.safetensors", + "gpt_neox.layers.18.input_layernorm.weight": "model-00005-of-00008.safetensors", + "gpt_neox.layers.18.mlp.dense_4h_to_h.bias": "model-00005-of-00008.safetensors", + "gpt_neox.layers.18.mlp.dense_4h_to_h.weight": "model-00005-of-00008.safetensors", + "gpt_neox.layers.18.mlp.dense_h_to_4h.bias": "model-00005-of-00008.safetensors", + "gpt_neox.layers.18.mlp.dense_h_to_4h.weight": "model-00005-of-00008.safetensors", + "gpt_neox.layers.18.post_attention_layernorm.bias": "model-00005-of-00008.safetensors", + "gpt_neox.layers.18.post_attention_layernorm.weight": "model-00005-of-00008.safetensors", + "gpt_neox.layers.19.attention.bias": "model-00005-of-00008.safetensors", + "gpt_neox.layers.19.attention.dense.bias": "model-00005-of-00008.safetensors", + "gpt_neox.layers.19.attention.dense.weight": "model-00005-of-00008.safetensors", + "gpt_neox.layers.19.attention.masked_bias": "model-00005-of-00008.safetensors", + "gpt_neox.layers.19.attention.query_key_value.bias": "model-00005-of-00008.safetensors", + "gpt_neox.layers.19.attention.query_key_value.weight": "model-00005-of-00008.safetensors", + "gpt_neox.layers.19.attention.rotary_emb.inv_freq": "model-00005-of-00008.safetensors", + "gpt_neox.layers.19.input_layernorm.bias": "model-00005-of-00008.safetensors", + "gpt_neox.layers.19.input_layernorm.weight": "model-00005-of-00008.safetensors", + "gpt_neox.layers.19.mlp.dense_4h_to_h.bias": "model-00005-of-00008.safetensors", + "gpt_neox.layers.19.mlp.dense_4h_to_h.weight": "model-00005-of-00008.safetensors", + "gpt_neox.layers.19.mlp.dense_h_to_4h.bias": "model-00005-of-00008.safetensors", + "gpt_neox.layers.19.mlp.dense_h_to_4h.weight": "model-00005-of-00008.safetensors", + "gpt_neox.layers.19.post_attention_layernorm.bias": "model-00005-of-00008.safetensors", + "gpt_neox.layers.19.post_attention_layernorm.weight": "model-00005-of-00008.safetensors", + "gpt_neox.layers.2.attention.bias": "model-00001-of-00008.safetensors", + "gpt_neox.layers.2.attention.dense.bias": "model-00001-of-00008.safetensors", + "gpt_neox.layers.2.attention.dense.weight": "model-00001-of-00008.safetensors", + "gpt_neox.layers.2.attention.masked_bias": "model-00001-of-00008.safetensors", + "gpt_neox.layers.2.attention.query_key_value.bias": "model-00001-of-00008.safetensors", + "gpt_neox.layers.2.attention.query_key_value.weight": "model-00001-of-00008.safetensors", + "gpt_neox.layers.2.attention.rotary_emb.inv_freq": "model-00001-of-00008.safetensors", + "gpt_neox.layers.2.input_layernorm.bias": "model-00001-of-00008.safetensors", + "gpt_neox.layers.2.input_layernorm.weight": "model-00001-of-00008.safetensors", + "gpt_neox.layers.2.mlp.dense_4h_to_h.bias": "model-00001-of-00008.safetensors", + "gpt_neox.layers.2.mlp.dense_4h_to_h.weight": "model-00001-of-00008.safetensors", + "gpt_neox.layers.2.mlp.dense_h_to_4h.bias": "model-00001-of-00008.safetensors", + "gpt_neox.layers.2.mlp.dense_h_to_4h.weight": "model-00001-of-00008.safetensors", + "gpt_neox.layers.2.post_attention_layernorm.bias": "model-00001-of-00008.safetensors", + "gpt_neox.layers.2.post_attention_layernorm.weight": "model-00001-of-00008.safetensors", + "gpt_neox.layers.20.attention.bias": "model-00005-of-00008.safetensors", + "gpt_neox.layers.20.attention.dense.bias": "model-00005-of-00008.safetensors", + "gpt_neox.layers.20.attention.dense.weight": "model-00005-of-00008.safetensors", + "gpt_neox.layers.20.attention.masked_bias": "model-00005-of-00008.safetensors", + "gpt_neox.layers.20.attention.query_key_value.bias": "model-00005-of-00008.safetensors", + "gpt_neox.layers.20.attention.query_key_value.weight": "model-00005-of-00008.safetensors", + "gpt_neox.layers.20.attention.rotary_emb.inv_freq": "model-00005-of-00008.safetensors", + "gpt_neox.layers.20.input_layernorm.bias": "model-00005-of-00008.safetensors", + "gpt_neox.layers.20.input_layernorm.weight": "model-00005-of-00008.safetensors", + "gpt_neox.layers.20.mlp.dense_4h_to_h.bias": "model-00006-of-00008.safetensors", + "gpt_neox.layers.20.mlp.dense_4h_to_h.weight": "model-00006-of-00008.safetensors", + "gpt_neox.layers.20.mlp.dense_h_to_4h.bias": "model-00005-of-00008.safetensors", + "gpt_neox.layers.20.mlp.dense_h_to_4h.weight": "model-00005-of-00008.safetensors", + "gpt_neox.layers.20.post_attention_layernorm.bias": "model-00005-of-00008.safetensors", + "gpt_neox.layers.20.post_attention_layernorm.weight": "model-00005-of-00008.safetensors", + "gpt_neox.layers.21.attention.bias": "model-00006-of-00008.safetensors", + "gpt_neox.layers.21.attention.dense.bias": "model-00006-of-00008.safetensors", + "gpt_neox.layers.21.attention.dense.weight": "model-00006-of-00008.safetensors", + "gpt_neox.layers.21.attention.masked_bias": "model-00006-of-00008.safetensors", + "gpt_neox.layers.21.attention.query_key_value.bias": "model-00006-of-00008.safetensors", + "gpt_neox.layers.21.attention.query_key_value.weight": "model-00006-of-00008.safetensors", + "gpt_neox.layers.21.attention.rotary_emb.inv_freq": "model-00006-of-00008.safetensors", + "gpt_neox.layers.21.input_layernorm.bias": "model-00006-of-00008.safetensors", + "gpt_neox.layers.21.input_layernorm.weight": "model-00006-of-00008.safetensors", + "gpt_neox.layers.21.mlp.dense_4h_to_h.bias": "model-00006-of-00008.safetensors", + "gpt_neox.layers.21.mlp.dense_4h_to_h.weight": "model-00006-of-00008.safetensors", + "gpt_neox.layers.21.mlp.dense_h_to_4h.bias": "model-00006-of-00008.safetensors", + "gpt_neox.layers.21.mlp.dense_h_to_4h.weight": "model-00006-of-00008.safetensors", + "gpt_neox.layers.21.post_attention_layernorm.bias": "model-00006-of-00008.safetensors", + "gpt_neox.layers.21.post_attention_layernorm.weight": "model-00006-of-00008.safetensors", + "gpt_neox.layers.22.attention.bias": "model-00006-of-00008.safetensors", + "gpt_neox.layers.22.attention.dense.bias": "model-00006-of-00008.safetensors", + "gpt_neox.layers.22.attention.dense.weight": "model-00006-of-00008.safetensors", + "gpt_neox.layers.22.attention.masked_bias": "model-00006-of-00008.safetensors", + "gpt_neox.layers.22.attention.query_key_value.bias": "model-00006-of-00008.safetensors", + "gpt_neox.layers.22.attention.query_key_value.weight": "model-00006-of-00008.safetensors", + "gpt_neox.layers.22.attention.rotary_emb.inv_freq": "model-00006-of-00008.safetensors", + "gpt_neox.layers.22.input_layernorm.bias": "model-00006-of-00008.safetensors", + "gpt_neox.layers.22.input_layernorm.weight": "model-00006-of-00008.safetensors", + "gpt_neox.layers.22.mlp.dense_4h_to_h.bias": "model-00006-of-00008.safetensors", + "gpt_neox.layers.22.mlp.dense_4h_to_h.weight": "model-00006-of-00008.safetensors", + "gpt_neox.layers.22.mlp.dense_h_to_4h.bias": "model-00006-of-00008.safetensors", + "gpt_neox.layers.22.mlp.dense_h_to_4h.weight": "model-00006-of-00008.safetensors", + "gpt_neox.layers.22.post_attention_layernorm.bias": "model-00006-of-00008.safetensors", + "gpt_neox.layers.22.post_attention_layernorm.weight": "model-00006-of-00008.safetensors", + "gpt_neox.layers.23.attention.bias": "model-00006-of-00008.safetensors", + "gpt_neox.layers.23.attention.dense.bias": "model-00006-of-00008.safetensors", + "gpt_neox.layers.23.attention.dense.weight": "model-00006-of-00008.safetensors", + "gpt_neox.layers.23.attention.masked_bias": "model-00006-of-00008.safetensors", + "gpt_neox.layers.23.attention.query_key_value.bias": "model-00006-of-00008.safetensors", + "gpt_neox.layers.23.attention.query_key_value.weight": "model-00006-of-00008.safetensors", + "gpt_neox.layers.23.attention.rotary_emb.inv_freq": "model-00006-of-00008.safetensors", + "gpt_neox.layers.23.input_layernorm.bias": "model-00006-of-00008.safetensors", + "gpt_neox.layers.23.input_layernorm.weight": "model-00006-of-00008.safetensors", + "gpt_neox.layers.23.mlp.dense_4h_to_h.bias": "model-00006-of-00008.safetensors", + "gpt_neox.layers.23.mlp.dense_4h_to_h.weight": "model-00006-of-00008.safetensors", + "gpt_neox.layers.23.mlp.dense_h_to_4h.bias": "model-00006-of-00008.safetensors", + "gpt_neox.layers.23.mlp.dense_h_to_4h.weight": "model-00006-of-00008.safetensors", + "gpt_neox.layers.23.post_attention_layernorm.bias": "model-00006-of-00008.safetensors", + "gpt_neox.layers.23.post_attention_layernorm.weight": "model-00006-of-00008.safetensors", + "gpt_neox.layers.24.attention.bias": "model-00006-of-00008.safetensors", + "gpt_neox.layers.24.attention.dense.bias": "model-00006-of-00008.safetensors", + "gpt_neox.layers.24.attention.dense.weight": "model-00006-of-00008.safetensors", + "gpt_neox.layers.24.attention.masked_bias": "model-00006-of-00008.safetensors", + "gpt_neox.layers.24.attention.query_key_value.bias": "model-00006-of-00008.safetensors", + "gpt_neox.layers.24.attention.query_key_value.weight": "model-00006-of-00008.safetensors", + "gpt_neox.layers.24.attention.rotary_emb.inv_freq": "model-00006-of-00008.safetensors", + "gpt_neox.layers.24.input_layernorm.bias": "model-00006-of-00008.safetensors", + "gpt_neox.layers.24.input_layernorm.weight": "model-00006-of-00008.safetensors", + "gpt_neox.layers.24.mlp.dense_4h_to_h.bias": "model-00006-of-00008.safetensors", + "gpt_neox.layers.24.mlp.dense_4h_to_h.weight": "model-00006-of-00008.safetensors", + "gpt_neox.layers.24.mlp.dense_h_to_4h.bias": "model-00006-of-00008.safetensors", + "gpt_neox.layers.24.mlp.dense_h_to_4h.weight": "model-00006-of-00008.safetensors", + "gpt_neox.layers.24.post_attention_layernorm.bias": "model-00006-of-00008.safetensors", + "gpt_neox.layers.24.post_attention_layernorm.weight": "model-00006-of-00008.safetensors", + "gpt_neox.layers.25.attention.bias": "model-00006-of-00008.safetensors", + "gpt_neox.layers.25.attention.dense.bias": "model-00007-of-00008.safetensors", + "gpt_neox.layers.25.attention.dense.weight": "model-00007-of-00008.safetensors", + "gpt_neox.layers.25.attention.masked_bias": "model-00006-of-00008.safetensors", + "gpt_neox.layers.25.attention.query_key_value.bias": "model-00007-of-00008.safetensors", + "gpt_neox.layers.25.attention.query_key_value.weight": "model-00007-of-00008.safetensors", + "gpt_neox.layers.25.attention.rotary_emb.inv_freq": "model-00006-of-00008.safetensors", + "gpt_neox.layers.25.input_layernorm.bias": "model-00006-of-00008.safetensors", + "gpt_neox.layers.25.input_layernorm.weight": "model-00006-of-00008.safetensors", + "gpt_neox.layers.25.mlp.dense_4h_to_h.bias": "model-00007-of-00008.safetensors", + "gpt_neox.layers.25.mlp.dense_4h_to_h.weight": "model-00007-of-00008.safetensors", + "gpt_neox.layers.25.mlp.dense_h_to_4h.bias": "model-00007-of-00008.safetensors", + "gpt_neox.layers.25.mlp.dense_h_to_4h.weight": "model-00007-of-00008.safetensors", + "gpt_neox.layers.25.post_attention_layernorm.bias": "model-00006-of-00008.safetensors", + "gpt_neox.layers.25.post_attention_layernorm.weight": "model-00006-of-00008.safetensors", + "gpt_neox.layers.26.attention.bias": "model-00007-of-00008.safetensors", + "gpt_neox.layers.26.attention.dense.bias": "model-00007-of-00008.safetensors", + "gpt_neox.layers.26.attention.dense.weight": "model-00007-of-00008.safetensors", + "gpt_neox.layers.26.attention.masked_bias": "model-00007-of-00008.safetensors", + "gpt_neox.layers.26.attention.query_key_value.bias": "model-00007-of-00008.safetensors", + "gpt_neox.layers.26.attention.query_key_value.weight": "model-00007-of-00008.safetensors", + "gpt_neox.layers.26.attention.rotary_emb.inv_freq": "model-00007-of-00008.safetensors", + "gpt_neox.layers.26.input_layernorm.bias": "model-00007-of-00008.safetensors", + "gpt_neox.layers.26.input_layernorm.weight": "model-00007-of-00008.safetensors", + "gpt_neox.layers.26.mlp.dense_4h_to_h.bias": "model-00007-of-00008.safetensors", + "gpt_neox.layers.26.mlp.dense_4h_to_h.weight": "model-00007-of-00008.safetensors", + "gpt_neox.layers.26.mlp.dense_h_to_4h.bias": "model-00007-of-00008.safetensors", + "gpt_neox.layers.26.mlp.dense_h_to_4h.weight": "model-00007-of-00008.safetensors", + "gpt_neox.layers.26.post_attention_layernorm.bias": "model-00007-of-00008.safetensors", + "gpt_neox.layers.26.post_attention_layernorm.weight": "model-00007-of-00008.safetensors", + "gpt_neox.layers.27.attention.bias": "model-00007-of-00008.safetensors", + "gpt_neox.layers.27.attention.dense.bias": "model-00007-of-00008.safetensors", + "gpt_neox.layers.27.attention.dense.weight": "model-00007-of-00008.safetensors", + "gpt_neox.layers.27.attention.masked_bias": "model-00007-of-00008.safetensors", + "gpt_neox.layers.27.attention.query_key_value.bias": "model-00007-of-00008.safetensors", + "gpt_neox.layers.27.attention.query_key_value.weight": "model-00007-of-00008.safetensors", + "gpt_neox.layers.27.attention.rotary_emb.inv_freq": "model-00007-of-00008.safetensors", + "gpt_neox.layers.27.input_layernorm.bias": "model-00007-of-00008.safetensors", + "gpt_neox.layers.27.input_layernorm.weight": "model-00007-of-00008.safetensors", + "gpt_neox.layers.27.mlp.dense_4h_to_h.bias": "model-00007-of-00008.safetensors", + "gpt_neox.layers.27.mlp.dense_4h_to_h.weight": "model-00007-of-00008.safetensors", + "gpt_neox.layers.27.mlp.dense_h_to_4h.bias": "model-00007-of-00008.safetensors", + "gpt_neox.layers.27.mlp.dense_h_to_4h.weight": "model-00007-of-00008.safetensors", + "gpt_neox.layers.27.post_attention_layernorm.bias": "model-00007-of-00008.safetensors", + "gpt_neox.layers.27.post_attention_layernorm.weight": "model-00007-of-00008.safetensors", + "gpt_neox.layers.28.attention.bias": "model-00007-of-00008.safetensors", + "gpt_neox.layers.28.attention.dense.bias": "model-00007-of-00008.safetensors", + "gpt_neox.layers.28.attention.dense.weight": "model-00007-of-00008.safetensors", + "gpt_neox.layers.28.attention.masked_bias": "model-00007-of-00008.safetensors", + "gpt_neox.layers.28.attention.query_key_value.bias": "model-00007-of-00008.safetensors", + "gpt_neox.layers.28.attention.query_key_value.weight": "model-00007-of-00008.safetensors", + "gpt_neox.layers.28.attention.rotary_emb.inv_freq": "model-00007-of-00008.safetensors", + "gpt_neox.layers.28.input_layernorm.bias": "model-00007-of-00008.safetensors", + "gpt_neox.layers.28.input_layernorm.weight": "model-00007-of-00008.safetensors", + "gpt_neox.layers.28.mlp.dense_4h_to_h.bias": "model-00007-of-00008.safetensors", + "gpt_neox.layers.28.mlp.dense_4h_to_h.weight": "model-00007-of-00008.safetensors", + "gpt_neox.layers.28.mlp.dense_h_to_4h.bias": "model-00007-of-00008.safetensors", + "gpt_neox.layers.28.mlp.dense_h_to_4h.weight": "model-00007-of-00008.safetensors", + "gpt_neox.layers.28.post_attention_layernorm.bias": "model-00007-of-00008.safetensors", + "gpt_neox.layers.28.post_attention_layernorm.weight": "model-00007-of-00008.safetensors", + "gpt_neox.layers.29.attention.bias": "model-00007-of-00008.safetensors", + "gpt_neox.layers.29.attention.dense.bias": "model-00007-of-00008.safetensors", + "gpt_neox.layers.29.attention.dense.weight": "model-00007-of-00008.safetensors", + "gpt_neox.layers.29.attention.masked_bias": "model-00007-of-00008.safetensors", + "gpt_neox.layers.29.attention.query_key_value.bias": "model-00007-of-00008.safetensors", + "gpt_neox.layers.29.attention.query_key_value.weight": "model-00007-of-00008.safetensors", + "gpt_neox.layers.29.attention.rotary_emb.inv_freq": "model-00007-of-00008.safetensors", + "gpt_neox.layers.29.input_layernorm.bias": "model-00007-of-00008.safetensors", + "gpt_neox.layers.29.input_layernorm.weight": "model-00007-of-00008.safetensors", + "gpt_neox.layers.29.mlp.dense_4h_to_h.bias": "model-00008-of-00008.safetensors", + "gpt_neox.layers.29.mlp.dense_4h_to_h.weight": "model-00008-of-00008.safetensors", + "gpt_neox.layers.29.mlp.dense_h_to_4h.bias": "model-00008-of-00008.safetensors", + "gpt_neox.layers.29.mlp.dense_h_to_4h.weight": "model-00008-of-00008.safetensors", + "gpt_neox.layers.29.post_attention_layernorm.bias": "model-00007-of-00008.safetensors", + "gpt_neox.layers.29.post_attention_layernorm.weight": "model-00007-of-00008.safetensors", + "gpt_neox.layers.3.attention.bias": "model-00001-of-00008.safetensors", + "gpt_neox.layers.3.attention.dense.bias": "model-00001-of-00008.safetensors", + "gpt_neox.layers.3.attention.dense.weight": "model-00001-of-00008.safetensors", + "gpt_neox.layers.3.attention.masked_bias": "model-00001-of-00008.safetensors", + "gpt_neox.layers.3.attention.query_key_value.bias": "model-00001-of-00008.safetensors", + "gpt_neox.layers.3.attention.query_key_value.weight": "model-00001-of-00008.safetensors", + "gpt_neox.layers.3.attention.rotary_emb.inv_freq": "model-00001-of-00008.safetensors", + "gpt_neox.layers.3.input_layernorm.bias": "model-00001-of-00008.safetensors", + "gpt_neox.layers.3.input_layernorm.weight": "model-00001-of-00008.safetensors", + "gpt_neox.layers.3.mlp.dense_4h_to_h.bias": "model-00002-of-00008.safetensors", + "gpt_neox.layers.3.mlp.dense_4h_to_h.weight": "model-00002-of-00008.safetensors", + "gpt_neox.layers.3.mlp.dense_h_to_4h.bias": "model-00002-of-00008.safetensors", + "gpt_neox.layers.3.mlp.dense_h_to_4h.weight": "model-00002-of-00008.safetensors", + "gpt_neox.layers.3.post_attention_layernorm.bias": "model-00001-of-00008.safetensors", + "gpt_neox.layers.3.post_attention_layernorm.weight": "model-00001-of-00008.safetensors", + "gpt_neox.layers.30.attention.bias": "model-00008-of-00008.safetensors", + "gpt_neox.layers.30.attention.dense.bias": "model-00008-of-00008.safetensors", + "gpt_neox.layers.30.attention.dense.weight": "model-00008-of-00008.safetensors", + "gpt_neox.layers.30.attention.masked_bias": "model-00008-of-00008.safetensors", + "gpt_neox.layers.30.attention.query_key_value.bias": "model-00008-of-00008.safetensors", + "gpt_neox.layers.30.attention.query_key_value.weight": "model-00008-of-00008.safetensors", + "gpt_neox.layers.30.attention.rotary_emb.inv_freq": "model-00008-of-00008.safetensors", + "gpt_neox.layers.30.input_layernorm.bias": "model-00008-of-00008.safetensors", + "gpt_neox.layers.30.input_layernorm.weight": "model-00008-of-00008.safetensors", + "gpt_neox.layers.30.mlp.dense_4h_to_h.bias": "model-00008-of-00008.safetensors", + "gpt_neox.layers.30.mlp.dense_4h_to_h.weight": "model-00008-of-00008.safetensors", + "gpt_neox.layers.30.mlp.dense_h_to_4h.bias": "model-00008-of-00008.safetensors", + "gpt_neox.layers.30.mlp.dense_h_to_4h.weight": "model-00008-of-00008.safetensors", + "gpt_neox.layers.30.post_attention_layernorm.bias": "model-00008-of-00008.safetensors", + "gpt_neox.layers.30.post_attention_layernorm.weight": "model-00008-of-00008.safetensors", + "gpt_neox.layers.31.attention.bias": "model-00008-of-00008.safetensors", + "gpt_neox.layers.31.attention.dense.bias": "model-00008-of-00008.safetensors", + "gpt_neox.layers.31.attention.dense.weight": "model-00008-of-00008.safetensors", + "gpt_neox.layers.31.attention.masked_bias": "model-00008-of-00008.safetensors", + "gpt_neox.layers.31.attention.query_key_value.bias": "model-00008-of-00008.safetensors", + "gpt_neox.layers.31.attention.query_key_value.weight": "model-00008-of-00008.safetensors", + "gpt_neox.layers.31.attention.rotary_emb.inv_freq": "model-00008-of-00008.safetensors", + "gpt_neox.layers.31.input_layernorm.bias": "model-00008-of-00008.safetensors", + "gpt_neox.layers.31.input_layernorm.weight": "model-00008-of-00008.safetensors", + "gpt_neox.layers.31.mlp.dense_4h_to_h.bias": "model-00008-of-00008.safetensors", + "gpt_neox.layers.31.mlp.dense_4h_to_h.weight": "model-00008-of-00008.safetensors", + "gpt_neox.layers.31.mlp.dense_h_to_4h.bias": "model-00008-of-00008.safetensors", + "gpt_neox.layers.31.mlp.dense_h_to_4h.weight": "model-00008-of-00008.safetensors", + "gpt_neox.layers.31.post_attention_layernorm.bias": "model-00008-of-00008.safetensors", + "gpt_neox.layers.31.post_attention_layernorm.weight": "model-00008-of-00008.safetensors", + "gpt_neox.layers.4.attention.bias": "model-00002-of-00008.safetensors", + "gpt_neox.layers.4.attention.dense.bias": "model-00002-of-00008.safetensors", + "gpt_neox.layers.4.attention.dense.weight": "model-00002-of-00008.safetensors", + "gpt_neox.layers.4.attention.masked_bias": "model-00002-of-00008.safetensors", + "gpt_neox.layers.4.attention.query_key_value.bias": "model-00002-of-00008.safetensors", + "gpt_neox.layers.4.attention.query_key_value.weight": "model-00002-of-00008.safetensors", + "gpt_neox.layers.4.attention.rotary_emb.inv_freq": "model-00002-of-00008.safetensors", + "gpt_neox.layers.4.input_layernorm.bias": "model-00002-of-00008.safetensors", + "gpt_neox.layers.4.input_layernorm.weight": "model-00002-of-00008.safetensors", + "gpt_neox.layers.4.mlp.dense_4h_to_h.bias": "model-00002-of-00008.safetensors", + "gpt_neox.layers.4.mlp.dense_4h_to_h.weight": "model-00002-of-00008.safetensors", + "gpt_neox.layers.4.mlp.dense_h_to_4h.bias": "model-00002-of-00008.safetensors", + "gpt_neox.layers.4.mlp.dense_h_to_4h.weight": "model-00002-of-00008.safetensors", + "gpt_neox.layers.4.post_attention_layernorm.bias": "model-00002-of-00008.safetensors", + "gpt_neox.layers.4.post_attention_layernorm.weight": "model-00002-of-00008.safetensors", + "gpt_neox.layers.5.attention.bias": "model-00002-of-00008.safetensors", + "gpt_neox.layers.5.attention.dense.bias": "model-00002-of-00008.safetensors", + "gpt_neox.layers.5.attention.dense.weight": "model-00002-of-00008.safetensors", + "gpt_neox.layers.5.attention.masked_bias": "model-00002-of-00008.safetensors", + "gpt_neox.layers.5.attention.query_key_value.bias": "model-00002-of-00008.safetensors", + "gpt_neox.layers.5.attention.query_key_value.weight": "model-00002-of-00008.safetensors", + "gpt_neox.layers.5.attention.rotary_emb.inv_freq": "model-00002-of-00008.safetensors", + "gpt_neox.layers.5.input_layernorm.bias": "model-00002-of-00008.safetensors", + "gpt_neox.layers.5.input_layernorm.weight": "model-00002-of-00008.safetensors", + "gpt_neox.layers.5.mlp.dense_4h_to_h.bias": "model-00002-of-00008.safetensors", + "gpt_neox.layers.5.mlp.dense_4h_to_h.weight": "model-00002-of-00008.safetensors", + "gpt_neox.layers.5.mlp.dense_h_to_4h.bias": "model-00002-of-00008.safetensors", + "gpt_neox.layers.5.mlp.dense_h_to_4h.weight": "model-00002-of-00008.safetensors", + "gpt_neox.layers.5.post_attention_layernorm.bias": "model-00002-of-00008.safetensors", + "gpt_neox.layers.5.post_attention_layernorm.weight": "model-00002-of-00008.safetensors", + "gpt_neox.layers.6.attention.bias": "model-00002-of-00008.safetensors", + "gpt_neox.layers.6.attention.dense.bias": "model-00002-of-00008.safetensors", + "gpt_neox.layers.6.attention.dense.weight": "model-00002-of-00008.safetensors", + "gpt_neox.layers.6.attention.masked_bias": "model-00002-of-00008.safetensors", + "gpt_neox.layers.6.attention.query_key_value.bias": "model-00002-of-00008.safetensors", + "gpt_neox.layers.6.attention.query_key_value.weight": "model-00002-of-00008.safetensors", + "gpt_neox.layers.6.attention.rotary_emb.inv_freq": "model-00002-of-00008.safetensors", + "gpt_neox.layers.6.input_layernorm.bias": "model-00002-of-00008.safetensors", + "gpt_neox.layers.6.input_layernorm.weight": "model-00002-of-00008.safetensors", + "gpt_neox.layers.6.mlp.dense_4h_to_h.bias": "model-00002-of-00008.safetensors", + "gpt_neox.layers.6.mlp.dense_4h_to_h.weight": "model-00002-of-00008.safetensors", + "gpt_neox.layers.6.mlp.dense_h_to_4h.bias": "model-00002-of-00008.safetensors", + "gpt_neox.layers.6.mlp.dense_h_to_4h.weight": "model-00002-of-00008.safetensors", + "gpt_neox.layers.6.post_attention_layernorm.bias": "model-00002-of-00008.safetensors", + "gpt_neox.layers.6.post_attention_layernorm.weight": "model-00002-of-00008.safetensors", + "gpt_neox.layers.7.attention.bias": "model-00002-of-00008.safetensors", + "gpt_neox.layers.7.attention.dense.bias": "model-00002-of-00008.safetensors", + "gpt_neox.layers.7.attention.dense.weight": "model-00002-of-00008.safetensors", + "gpt_neox.layers.7.attention.masked_bias": "model-00002-of-00008.safetensors", + "gpt_neox.layers.7.attention.query_key_value.bias": "model-00002-of-00008.safetensors", + "gpt_neox.layers.7.attention.query_key_value.weight": "model-00002-of-00008.safetensors", + "gpt_neox.layers.7.attention.rotary_emb.inv_freq": "model-00002-of-00008.safetensors", + "gpt_neox.layers.7.input_layernorm.bias": "model-00002-of-00008.safetensors", + "gpt_neox.layers.7.input_layernorm.weight": "model-00002-of-00008.safetensors", + "gpt_neox.layers.7.mlp.dense_4h_to_h.bias": "model-00003-of-00008.safetensors", + "gpt_neox.layers.7.mlp.dense_4h_to_h.weight": "model-00003-of-00008.safetensors", + "gpt_neox.layers.7.mlp.dense_h_to_4h.bias": "model-00002-of-00008.safetensors", + "gpt_neox.layers.7.mlp.dense_h_to_4h.weight": "model-00002-of-00008.safetensors", + "gpt_neox.layers.7.post_attention_layernorm.bias": "model-00002-of-00008.safetensors", + "gpt_neox.layers.7.post_attention_layernorm.weight": "model-00002-of-00008.safetensors", + "gpt_neox.layers.8.attention.bias": "model-00003-of-00008.safetensors", + "gpt_neox.layers.8.attention.dense.bias": "model-00003-of-00008.safetensors", + "gpt_neox.layers.8.attention.dense.weight": "model-00003-of-00008.safetensors", + "gpt_neox.layers.8.attention.masked_bias": "model-00003-of-00008.safetensors", + "gpt_neox.layers.8.attention.query_key_value.bias": "model-00003-of-00008.safetensors", + "gpt_neox.layers.8.attention.query_key_value.weight": "model-00003-of-00008.safetensors", + "gpt_neox.layers.8.attention.rotary_emb.inv_freq": "model-00003-of-00008.safetensors", + "gpt_neox.layers.8.input_layernorm.bias": "model-00003-of-00008.safetensors", + "gpt_neox.layers.8.input_layernorm.weight": "model-00003-of-00008.safetensors", + "gpt_neox.layers.8.mlp.dense_4h_to_h.bias": "model-00003-of-00008.safetensors", + "gpt_neox.layers.8.mlp.dense_4h_to_h.weight": "model-00003-of-00008.safetensors", + "gpt_neox.layers.8.mlp.dense_h_to_4h.bias": "model-00003-of-00008.safetensors", + "gpt_neox.layers.8.mlp.dense_h_to_4h.weight": "model-00003-of-00008.safetensors", + "gpt_neox.layers.8.post_attention_layernorm.bias": "model-00003-of-00008.safetensors", + "gpt_neox.layers.8.post_attention_layernorm.weight": "model-00003-of-00008.safetensors", + "gpt_neox.layers.9.attention.bias": "model-00003-of-00008.safetensors", + "gpt_neox.layers.9.attention.dense.bias": "model-00003-of-00008.safetensors", + "gpt_neox.layers.9.attention.dense.weight": "model-00003-of-00008.safetensors", + "gpt_neox.layers.9.attention.masked_bias": "model-00003-of-00008.safetensors", + "gpt_neox.layers.9.attention.query_key_value.bias": "model-00003-of-00008.safetensors", + "gpt_neox.layers.9.attention.query_key_value.weight": "model-00003-of-00008.safetensors", + "gpt_neox.layers.9.attention.rotary_emb.inv_freq": "model-00003-of-00008.safetensors", + "gpt_neox.layers.9.input_layernorm.bias": "model-00003-of-00008.safetensors", + "gpt_neox.layers.9.input_layernorm.weight": "model-00003-of-00008.safetensors", + "gpt_neox.layers.9.mlp.dense_4h_to_h.bias": "model-00003-of-00008.safetensors", + "gpt_neox.layers.9.mlp.dense_4h_to_h.weight": "model-00003-of-00008.safetensors", + "gpt_neox.layers.9.mlp.dense_h_to_4h.bias": "model-00003-of-00008.safetensors", + "gpt_neox.layers.9.mlp.dense_h_to_4h.weight": "model-00003-of-00008.safetensors", + "gpt_neox.layers.9.post_attention_layernorm.bias": "model-00003-of-00008.safetensors", + "gpt_neox.layers.9.post_attention_layernorm.weight": "model-00003-of-00008.safetensors" + } +} diff --git a/pytorch_model.bin b/pytorch_model.bin new file mode 100644 index 0000000..7cd0fbe --- /dev/null +++ b/pytorch_model.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c690d2e06597fb5f5f7c0e7444d417fe9f391787741a26b1c0fd953b0bdfbd6f +size 7754324845 diff --git a/special_tokens_map.json b/special_tokens_map.json new file mode 100644 index 0000000..b0b38c1 --- /dev/null +++ b/special_tokens_map.json @@ -0,0 +1,11 @@ +{ + "additional_special_tokens": [ + "<|endoftext|>", + "<|sep|>", + "<|acc|>", + "<|tel|>", + "<|rrn|>" + ], + "eos_token": "<|endoftext|>", + "pad_token": "<|endoftext|>" +} diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..379a60e --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:60e8bd123994badd563a603bce1a319dd7f5dc4798a476c2d88c671766db2f24 +size 1652158 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..8459768 --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,6 @@ +{ + "name_or_path": "EleutherAI/polyglot-ko-3.8b", + "eos_token": "<|endoftext|>", + "pad_token": "<|endoftext|>", + "tokenizer_class": "PreTrainedTokenizerFast" +}