commit 5074d5ec8b32122056af19548d99d95d4d04fac7 Author: ModelHub XC Date: Sat May 23 05:18:12 2026 +0800 初始化项目，由ModelHub XC社区提供模型 Model: EleutherAI/polyglot-ko-5.8b Source: Original Platform diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..21b3632 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,49 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bin.* filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zstandard filter=lfs diff=lfs merge=lfs -text +*.tfevents* filter=lfs diff=lfs merge=lfs -text +*.db* filter=lfs diff=lfs merge=lfs -text +*.ark* filter=lfs diff=lfs merge=lfs -text +**/*ckpt*data* filter=lfs diff=lfs merge=lfs -text +**/*ckpt*.meta filter=lfs diff=lfs merge=lfs -text +**/*ckpt*.index filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.gguf* filter=lfs diff=lfs merge=lfs -text +*.ggml filter=lfs diff=lfs merge=lfs -text +*.llamafile* filter=lfs diff=lfs merge=lfs -text +*.pt2 filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text + +tokenizer.json filter=lfs diff=lfs merge=lfs -text \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..98823a6 --- /dev/null +++ b/README.md @@ -0,0 +1,200 @@ +--- +language: +- ko +tags: +- pytorch +- causal-lm +license: apache-2.0 + +--- +# Polyglot-Ko-5.8B + +## Model Description +Polyglot-Ko is a series of large-scale Korean autoregressive language models made by the EleutherAI polyglot team. + +| Hyperparameter | Value | +|----------------------|----------------------------------------------------------------------------------------------------------------------------------------| +| \\(n_{parameters}\\) | 5,885,059,072 | +| \\(n_{layers}\\) | 28 | +| \\(d_{model}\\) | 4096 | +| \\(d_{ff}\\) | 16,384 | +| \\(n_{heads}\\) | 16 | +| \\(d_{head}\\) | 256 | +| \\(n_{ctx}\\) | 2,048 | +| \\(n_{vocab}\\) | 30,003 / 30,080 | +| Positional Encoding | [Rotary Position Embedding (RoPE)](https://arxiv.org/abs/2104.09864) | +| RoPE Dimensions | [64](https://github.com/kingoflolz/mesh-transformer-jax/blob/f2aa66e0925de6593dcbb70e72399b97b4130482/mesh_transformer/layers.py#L223) | + +The model consists of 28 transformer layers with a model dimension of 4096, and a feedforward dimension of 16384. The model +dimension is split into 16 heads, each with a dimension of 256. Rotary Position Embedding (RoPE) is applied to 64 +dimensions of each head. The model is trained with a tokenization vocabulary of 30003. + +## Training data + +Polyglot-Ko-5.8B was trained on 863 GB of Korean language data (1.2TB before processing), a large-scale dataset curated by [TUNiB](https://tunib.ai/). The data collection process has abided by South Korean laws. This dataset was collected for the purpose of training Polyglot-Ko models, so it will not be released for public use. + +| Source |Size (GB) | Link | +|-------------------------------------|---------|------------------------------------------| +| Korean blog posts | 682.3 | - | +| Korean news dataset | 87.0 | - | +| Modu corpus | 26.4 |corpus.korean.go.kr | +| Korean patent dataset | 19.0 | - | +| Korean Q & A dataset | 18.1 | - | +| KcBert dataset | 12.7 | github.com/Beomi/KcBERT | +| Korean fiction dataset | 6.1 | - | +| Korean online comments | 4.2 | - | +| Korean wikipedia | 1.4 | ko.wikipedia.org | +| Clova call | < 1.0 | github.com/clovaai/ClovaCall | +| Naver sentiment movie corpus | < 1.0 | github.com/e9t/nsmc | +| Korean hate speech dataset | < 1.0 | - | +| Open subtitles | < 1.0 | opus.nlpl.eu/OpenSubtitles.php | +| AIHub various tasks datasets | < 1.0 |aihub.or.kr | +| Standard Korean language dictionary | < 1.0 | stdict.korean.go.kr/main/main.do | + +Furthermore, in order to avoid the model memorizing and generating personally identifiable information (PII) in the training data, we masked out the following sensitive information in the pre-processing stage: + +* `<|acc|>` : bank account number +* `<|rrn|>` : resident registration number +* `<|tell|>` : phone number + +## Training procedure +Polyglot-Ko-5.8B was trained for 172 billion tokens over 320,000 steps on 256 A100 GPUs with the [GPT-NeoX framework](https://github.com/EleutherAI/gpt-neox). It was trained as an autoregressive language model, using cross-entropy loss to maximize the likelihood of predicting the next token. + +## How to use + +This model can be easily loaded using the `AutoModelForCausalLM` class: + +```python +from transformers import AutoTokenizer, AutoModelForCausalLM + +tokenizer = AutoTokenizer.from_pretrained("EleutherAI/polyglot-ko-5.8b") +model = AutoModelForCausalLM.from_pretrained("EleutherAI/polyglot-ko-5.8b") +``` + +## Evaluation results + +We evaluate Polyglot-Ko-3.8B on [KOBEST dataset](https://arxiv.org/abs/2204.04541), a benchmark with 5 downstream tasks, against comparable models such as skt/ko-gpt-trinity-1.2B-v0.5, kakaobrain/kogpt and facebook/xglm-7.5B, using the prompts provided in the paper. + +The following tables show the results when the number of few-shot examples differ. You can reproduce these results using the [polyglot branch of lm-evaluation-harness](https://github.com/EleutherAI/lm-evaluation-harness/tree/polyglot) and the following scripts. For a fair comparison, all models were run under the same conditions and using the same prompts. In the tables, `n` refers to the number of few-shot examples. + +In case of WiC dataset, all models show random performance. + +```console +python main.py \ + --model gpt2 \ + --model_args pretrained='EleutherAI/polyglot-ko-3.8b' \ + --tasks kobest_copa,kobest_hellaswag \ + --num_fewshot $YOUR_NUM_FEWSHOT \ + --batch_size $YOUR_BATCH_SIZE \ + --device $YOUR_DEVICE \ + --output_path $/path/to/output/ +``` + +### COPA (F1) + +| Model | params | 0-shot | 5-shot | 10-shot | 50-shot | +|----------------------------------------------------------------------------------------------|--------|--------|--------|---------|---------| +| [skt/ko-gpt-trinity-1.2B-v0.5](https://huggingface.co/skt/ko-gpt-trinity-1.2B-v0.5) | 1.2B | 0.6696 | 0.6477 | 0.6419 | 0.6514 | +| [kakaobrain/kogpt](https://huggingface.co/kakaobrain/kogpt) | 6.0B | 0.7345 | 0.7287 | 0.7277 | 0.7479 | +| [facebook/xglm-7.5B](https://huggingface.co/facebook/xglm-7.5B) | 7.5B | 0.6723 | 0.6731 | 0.6769 | 0.7119 | +| [EleutherAI/polyglot-ko-1.3b](https://huggingface.co/EleutherAI/polyglot-ko-1.3b) | 1.3B | 0.7196 | 0.7193 | 0.7204 | 0.7206 | +| [EleutherAI/polyglot-ko-3.8b](https://huggingface.co/EleutherAI/polyglot-ko-3.8b) | 3.8B | 0.7595 | 0.7608 | 0.7638 | 0.7788 | +| **[EleutherAI/polyglot-ko-5.8b](https://huggingface.co/EleutherAI/polyglot-ko-5.8b) (this)** | **5.8B** | **0.7745** | **0.7676** | **0.7775** | **0.7887** | +| [EleutherAI/polyglot-ko-12.8b](https://huggingface.co/EleutherAI/polyglot-ko-12.8b) | 12.8B | 0.7937 | 0.8108 | 0.8037 | 0.8369 | + + + +### HellaSwag (F1) + +| Model | params | 0-shot | 5-shot | 10-shot | 50-shot | +|----------------------------------------------------------------------------------------------|--------|--------|--------|---------|---------| +| [skt/ko-gpt-trinity-1.2B-v0.5](https://huggingface.co/skt/ko-gpt-trinity-1.2B-v0.5) | 1.2B | 0.5243 | 0.5272 | 0.5166 | 0.5352 | +| [kakaobrain/kogpt](https://huggingface.co/kakaobrain/kogpt) | 6.0B | 0.5590 | 0.5833 | 0.5828 | 0.5907 | +| [facebook/xglm-7.5B](https://huggingface.co/facebook/xglm-7.5B) | 7.5B | 0.5665 | 0.5689 | 0.5565 | 0.5622 | +| [EleutherAI/polyglot-ko-1.3b](https://huggingface.co/EleutherAI/polyglot-ko-1.3b) | 1.3B | 0.5247 | 0.5260 | 0.5278 | 0.5427 | +| [EleutherAI/polyglot-ko-3.8b](https://huggingface.co/EleutherAI/polyglot-ko-3.8b) | 3.8B | 0.5707 | 0.5830 | 0.5670 | 0.5787 | +| **[EleutherAI/polyglot-ko-5.8b](https://huggingface.co/EleutherAI/polyglot-ko-5.8b) (this)** | **5.8B** | **0.5976** | **0.5998** | **0.5979** | **0.6208** | +| [EleutherAI/polyglot-ko-12.8b](https://huggingface.co/EleutherAI/polyglot-ko-12.8b) | 12.8B | 0.5954 | 0.6306 | 0.6098 | 0.6118 | + + + +### BoolQ (F1) + +| Model | params | 0-shot | 5-shot | 10-shot | 50-shot | +|----------------------------------------------------------------------------------------------|--------|--------|--------|---------|---------| +| [skt/ko-gpt-trinity-1.2B-v0.5](https://huggingface.co/skt/ko-gpt-trinity-1.2B-v0.5) | 1.2B | 0.3356 | 0.4014 | 0.3640 | 0.3560 | +| [kakaobrain/kogpt](https://huggingface.co/kakaobrain/kogpt) | 6.0B | 0.4514 | 0.5981 | 0.5499 | 0.5202 | +| [facebook/xglm-7.5B](https://huggingface.co/facebook/xglm-7.5B) | 7.5B | 0.4464 | 0.3324 | 0.3324 | 0.3324 | +| [EleutherAI/polyglot-ko-1.3b](https://huggingface.co/EleutherAI/polyglot-ko-1.3b) | 1.3B | 0.3552 | 0.4751 | 0.4109 | 0.4038 | +| [EleutherAI/polyglot-ko-3.8b](https://huggingface.co/EleutherAI/polyglot-ko-3.8b) | 3.8B | 0.4320 | 0.5263 | 0.4930 | 0.4038 | +| **[EleutherAI/polyglot-ko-5.8b](https://huggingface.co/EleutherAI/polyglot-ko-5.8b) (this)** | **5.8B** | **0.4356** | **0.5698** | **0.5187** | **0.5236** | +| [EleutherAI/polyglot-ko-12.8b](https://huggingface.co/EleutherAI/polyglot-ko-12.8b) | 12.8B | 0.4818 | 0.6041 | 0.6289 | 0.6448 | + + + +### SentiNeg (F1) + +| Model | params | 0-shot | 5-shot | 10-shot | 50-shot | +|----------------------------------------------------------------------------------------------|--------|--------|--------|---------|---------| +| [skt/ko-gpt-trinity-1.2B-v0.5](https://huggingface.co/skt/ko-gpt-trinity-1.2B-v0.5) | 1.2B | 0.6065 | 0.6878 | 0.7280 | 0.8413 | +| [kakaobrain/kogpt](https://huggingface.co/kakaobrain/kogpt) | 6.0B | 0.3747 | 0.8942 | 0.9294 | 0.9698 | +| [facebook/xglm-7.5B](https://huggingface.co/facebook/xglm-7.5B) | 7.5B | 0.3578 | 0.4471 | 0.3964 | 0.5271 | +| [EleutherAI/polyglot-ko-1.3b](https://huggingface.co/EleutherAI/polyglot-ko-1.3b) | 1.3B | 0.6790 | 0.6257 | 0.5514 | 0.7851 | +| [EleutherAI/polyglot-ko-3.8b](https://huggingface.co/EleutherAI/polyglot-ko-3.8b) | 3.8B | 0.4858 | 0.7950 | 0.7320 | 0.7851 | +| **[EleutherAI/polyglot-ko-5.8b](https://huggingface.co/EleutherAI/polyglot-ko-5.8b) (this)** | **5.8B** | **0.3394** | **0.8841** | **0.8808** | **0.9521** | +| [EleutherAI/polyglot-ko-12.8b](https://huggingface.co/EleutherAI/polyglot-ko-12.8b) | 12.8B | 0.9117 | 0.9015 | 0.9345 | 0.9723 | + + + +### WiC (F1) + +| Model | params | 0-shot | 5-shot | 10-shot | 50-shot | +|----------------------------------------------------------------------------------------------|--------|--------|--------|---------|---------| +| [skt/ko-gpt-trinity-1.2B-v0.5](https://huggingface.co/skt/ko-gpt-trinity-1.2B-v0.5) | 1.2B | 0.3290 | 0.4313 | 0.4001 | 0.3621 | +| [kakaobrain/kogpt](https://huggingface.co/kakaobrain/kogpt) | 6.0B | 0.3526 | 0.4775 | 0.4358 | 0.4061 | +| [facebook/xglm-7.5B](https://huggingface.co/facebook/xglm-7.5B) | 7.5B | 0.3280 | 0.4903 | 0.4945 | 0.3656 | +| [EleutherAI/polyglot-ko-1.3b](https://huggingface.co/EleutherAI/polyglot-ko-1.3b) | 1.3B | 0.3297 | 0.4850 | 0.4650 | 0.3290 | +| [EleutherAI/polyglot-ko-3.8b](https://huggingface.co/EleutherAI/polyglot-ko-3.8b) | 3.8B | 0.3390 | 0.4944 | 0.4203 | 0.3835 | +| **[EleutherAI/polyglot-ko-5.8b](https://huggingface.co/EleutherAI/polyglot-ko-5.8b) (this)** | **5.8B** | **0.3913** | **0.4688** | **0.4189** | **0.3910** | +| [EleutherAI/polyglot-ko-12.8b](https://huggingface.co/EleutherAI/polyglot-ko-12.8b) | 12.8B | 0.3985 | 0.3683 | 0.3307 | 0.3273 | + + + +## Limitations and Biases + +Polyglot-Ko has been trained to optimize next token prediction. Language models such as this are often used for a wide variety of tasks and it is important to be aware of possible unexpected outcomes. For instance, Polyglot-Ko will not always return the most factual or accurate response but the most statistically likely one. In addition, Polyglot may produce socially unacceptable or offensive content. We recommend having a human curator or other filtering mechanism to censor sensitive content. + +## Citation and Related Information +### BibTeX entry +If you find our work useful, please consider citing: +```bibtex +@misc{ko2023technical, + title={A Technical Report for Polyglot-Ko: Open-Source Large-Scale Korean Language Models}, + author={Hyunwoong Ko and Kichang Yang and Minho Ryu and Taekyoon Choi and Seungmu Yang and jiwung Hyun and Sungho Park}, + year={2023}, + eprint={2306.02254}, + archivePrefix={arXiv}, + primaryClass={cs.CL} +} +``` + +### Licensing +All our models are licensed under the terms of the Apache License 2.0. + +``` +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +``` + +### Acknowledgement + +This project was made possible thanks to the computing resources from [Stability.ai](https://stability.ai), and thanks to [TUNiB](https://tunib.ai) for providing a large-scale Korean dataset for this work. diff --git a/config.json b/config.json new file mode 100644 index 0000000..abea5e5 --- /dev/null +++ b/config.json @@ -0,0 +1,27 @@ +{ + "_name_or_path": "./polyglot-ko-5.8b/", + "architectures": [ + "GPTNeoXForCausalLM" + ], + "bos_token_id": 0, + "classifier_dropout": 0.1, + "eos_token_id": 2, + "hidden_act": "gelu", + "hidden_size": 4096, + "initializer_range": 0.02, + "intermediate_size": 16384, + "layer_norm_eps": 1e-05, + "max_position_embeddings": 2048, + "model_type": "gpt_neox", + "num_attention_heads": 16, + "num_hidden_layers": 28, + "num_steps": "global_step320000", + "rotary_emb_base": 10000, + "rotary_pct": 0.25, + "tie_word_embeddings": false, + "torch_dtype": "float16", + "transformers_version": "4.29.2", + "use_cache": true, + "use_parallel_residual": true, + "vocab_size": 30080 +} diff --git a/configuration.json b/configuration.json new file mode 100644 index 0000000..bbeeda1 --- /dev/null +++ b/configuration.json @@ -0,0 +1 @@ +{"framework": "pytorch", "task": "text-generation", "allow_remote": true} \ No newline at end of file diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..7a838ce --- /dev/null +++ b/generation_config.json @@ -0,0 +1,6 @@ +{ + "_from_model_config": true, + "bos_token_id": 0, + "eos_token_id": 2, + "transformers_version": "4.29.2" +} diff --git a/model-00001-of-00013.safetensors b/model-00001-of-00013.safetensors new file mode 100644 index 0000000..a1d9dc8 --- /dev/null +++ b/model-00001-of-00013.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9525253224c8c49f441d63f85b4e41b042865dd15309c9f75c6522b502b951f5 +size 926101004 diff --git a/model-00002-of-00013.safetensors b/model-00002-of-00013.safetensors new file mode 100644 index 0000000..4e190b5 --- /dev/null +++ b/model-00002-of-00013.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4faf44378eac1c02c3d569c74d9d95c2fe230aa0cc2a1901417e92f870f21e89 +size 952365814 diff --git a/model-00003-of-00013.safetensors b/model-00003-of-00013.safetensors new file mode 100644 index 0000000..de47c6b --- /dev/null +++ b/model-00003-of-00013.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:389cd21e9d44efd6952a3ba127a704538ff895163c7d303a120988d290543ac2 +size 948162644 diff --git a/model-00004-of-00013.safetensors b/model-00004-of-00013.safetensors new file mode 100644 index 0000000..bf9c4a4 --- /dev/null +++ b/model-00004-of-00013.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e9a7391a3a23ee3d11d4ae2b97b0d5f6be18c2e3a7507f19a8645be8e9014495 +size 948162412 diff --git a/model-00005-of-00013.safetensors b/model-00005-of-00013.safetensors new file mode 100644 index 0000000..a80d7b5 --- /dev/null +++ b/model-00005-of-00013.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:377e2ea5aa0f07e1d7cd7a55b67656123fe47fc9b88873b76a9315c71c111066 +size 952365830 diff --git a/model-00006-of-00013.safetensors b/model-00006-of-00013.safetensors new file mode 100644 index 0000000..906749a --- /dev/null +++ b/model-00006-of-00013.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:24f189ce457820612a49e88b536db63c4b132f37a850ca1f47728392ae3f436c +size 948162676 diff --git a/model-00007-of-00013.safetensors b/model-00007-of-00013.safetensors new file mode 100644 index 0000000..241c069 --- /dev/null +++ b/model-00007-of-00013.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e1a04027c907c186cddecb4779fa3d2b0adc9757dab6e51ce1b0603135176454 +size 948162444 diff --git a/model-00008-of-00013.safetensors b/model-00008-of-00013.safetensors new file mode 100644 index 0000000..64b0412 --- /dev/null +++ b/model-00008-of-00013.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:081aff38df82775822de25a7c9382735a82d4e24acb3fb4dc275ae29a5a74b88 +size 952365854 diff --git a/model-00009-of-00013.safetensors b/model-00009-of-00013.safetensors new file mode 100644 index 0000000..7f83317 --- /dev/null +++ b/model-00009-of-00013.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:039fff3bb0d4e39cae9f0c5a0dda8c4b1dfbff00bf7036041f5f453fb4047663 +size 948162676 diff --git a/model-00010-of-00013.safetensors b/model-00010-of-00013.safetensors new file mode 100644 index 0000000..3be8a71 --- /dev/null +++ b/model-00010-of-00013.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4628cac9f442d663d7ea8d4535341fb6bbc446e76394c82eab97b0d89140c6f5 +size 948162444 diff --git a/model-00011-of-00013.safetensors b/model-00011-of-00013.safetensors new file mode 100644 index 0000000..4fa9e7c --- /dev/null +++ b/model-00011-of-00013.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dc08f6c4e98e71f106cc236c45f60f4f94845c106c26c760e0b3635b2e387a26 +size 952365854 diff --git a/model-00012-of-00013.safetensors b/model-00012-of-00013.safetensors new file mode 100644 index 0000000..fa1aeea --- /dev/null +++ b/model-00012-of-00013.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:047fb3056bff118af349fe554aa3fb165f389632ac0e44722ea5c7aef9e00b51 +size 948162676 diff --git a/model-00013-of-00013.safetensors b/model-00013-of-00013.safetensors new file mode 100644 index 0000000..0681e87 --- /dev/null +++ b/model-00013-of-00013.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6865c608008eba473353904c4f4b8d23a413617f9a0c3b22257c7380cdbfcbca +size 514908960 diff --git a/model.safetensors.index.json b/model.safetensors.index.json new file mode 100644 index 0000000..dfc8fd8 --- /dev/null +++ b/model.safetensors.index.json @@ -0,0 +1,431 @@ +{ + "metadata": { + "total_size": 11784801848.0 + }, + "weight_map": { + "embed_out.weight": "model-00013-of-00013.safetensors", + "gpt_neox.embed_in.weight": "model-00001-of-00013.safetensors", + "gpt_neox.final_layer_norm.bias": "model-00013-of-00013.safetensors", + "gpt_neox.final_layer_norm.weight": "model-00013-of-00013.safetensors", + "gpt_neox.layers.0.attention.bias": "model-00001-of-00013.safetensors", + "gpt_neox.layers.0.attention.dense.bias": "model-00001-of-00013.safetensors", + "gpt_neox.layers.0.attention.dense.weight": "model-00001-of-00013.safetensors", + "gpt_neox.layers.0.attention.masked_bias": "model-00001-of-00013.safetensors", + "gpt_neox.layers.0.attention.query_key_value.bias": "model-00001-of-00013.safetensors", + "gpt_neox.layers.0.attention.query_key_value.weight": "model-00001-of-00013.safetensors", + "gpt_neox.layers.0.attention.rotary_emb.inv_freq": "model-00001-of-00013.safetensors", + "gpt_neox.layers.0.input_layernorm.bias": "model-00001-of-00013.safetensors", + "gpt_neox.layers.0.input_layernorm.weight": "model-00001-of-00013.safetensors", + "gpt_neox.layers.0.mlp.dense_4h_to_h.bias": "model-00001-of-00013.safetensors", + "gpt_neox.layers.0.mlp.dense_4h_to_h.weight": "model-00001-of-00013.safetensors", + "gpt_neox.layers.0.mlp.dense_h_to_4h.bias": "model-00001-of-00013.safetensors", + "gpt_neox.layers.0.mlp.dense_h_to_4h.weight": "model-00001-of-00013.safetensors", + "gpt_neox.layers.0.post_attention_layernorm.bias": "model-00001-of-00013.safetensors", + "gpt_neox.layers.0.post_attention_layernorm.weight": "model-00001-of-00013.safetensors", + "gpt_neox.layers.1.attention.bias": "model-00001-of-00013.safetensors", + "gpt_neox.layers.1.attention.dense.bias": "model-00001-of-00013.safetensors", + "gpt_neox.layers.1.attention.dense.weight": "model-00001-of-00013.safetensors", + "gpt_neox.layers.1.attention.masked_bias": "model-00001-of-00013.safetensors", + "gpt_neox.layers.1.attention.query_key_value.bias": "model-00001-of-00013.safetensors", + "gpt_neox.layers.1.attention.query_key_value.weight": "model-00001-of-00013.safetensors", + "gpt_neox.layers.1.attention.rotary_emb.inv_freq": "model-00001-of-00013.safetensors", + "gpt_neox.layers.1.input_layernorm.bias": "model-00001-of-00013.safetensors", + "gpt_neox.layers.1.input_layernorm.weight": "model-00001-of-00013.safetensors", + "gpt_neox.layers.1.mlp.dense_4h_to_h.bias": "model-00002-of-00013.safetensors", + "gpt_neox.layers.1.mlp.dense_4h_to_h.weight": "model-00002-of-00013.safetensors", + "gpt_neox.layers.1.mlp.dense_h_to_4h.bias": "model-00001-of-00013.safetensors", + "gpt_neox.layers.1.mlp.dense_h_to_4h.weight": "model-00001-of-00013.safetensors", + "gpt_neox.layers.1.post_attention_layernorm.bias": "model-00001-of-00013.safetensors", + "gpt_neox.layers.1.post_attention_layernorm.weight": "model-00001-of-00013.safetensors", + "gpt_neox.layers.10.attention.bias": "model-00005-of-00013.safetensors", + "gpt_neox.layers.10.attention.dense.bias": "model-00005-of-00013.safetensors", + "gpt_neox.layers.10.attention.dense.weight": "model-00005-of-00013.safetensors", + "gpt_neox.layers.10.attention.masked_bias": "model-00005-of-00013.safetensors", + "gpt_neox.layers.10.attention.query_key_value.bias": "model-00005-of-00013.safetensors", + "gpt_neox.layers.10.attention.query_key_value.weight": "model-00005-of-00013.safetensors", + "gpt_neox.layers.10.attention.rotary_emb.inv_freq": "model-00005-of-00013.safetensors", + "gpt_neox.layers.10.input_layernorm.bias": "model-00005-of-00013.safetensors", + "gpt_neox.layers.10.input_layernorm.weight": "model-00005-of-00013.safetensors", + "gpt_neox.layers.10.mlp.dense_4h_to_h.bias": "model-00005-of-00013.safetensors", + "gpt_neox.layers.10.mlp.dense_4h_to_h.weight": "model-00005-of-00013.safetensors", + "gpt_neox.layers.10.mlp.dense_h_to_4h.bias": "model-00005-of-00013.safetensors", + "gpt_neox.layers.10.mlp.dense_h_to_4h.weight": "model-00005-of-00013.safetensors", + "gpt_neox.layers.10.post_attention_layernorm.bias": "model-00005-of-00013.safetensors", + "gpt_neox.layers.10.post_attention_layernorm.weight": "model-00005-of-00013.safetensors", + "gpt_neox.layers.11.attention.bias": "model-00005-of-00013.safetensors", + "gpt_neox.layers.11.attention.dense.bias": "model-00006-of-00013.safetensors", + "gpt_neox.layers.11.attention.dense.weight": "model-00006-of-00013.safetensors", + "gpt_neox.layers.11.attention.masked_bias": "model-00005-of-00013.safetensors", + "gpt_neox.layers.11.attention.query_key_value.bias": "model-00006-of-00013.safetensors", + "gpt_neox.layers.11.attention.query_key_value.weight": "model-00006-of-00013.safetensors", + "gpt_neox.layers.11.attention.rotary_emb.inv_freq": "model-00005-of-00013.safetensors", + "gpt_neox.layers.11.input_layernorm.bias": "model-00005-of-00013.safetensors", + "gpt_neox.layers.11.input_layernorm.weight": "model-00005-of-00013.safetensors", + "gpt_neox.layers.11.mlp.dense_4h_to_h.bias": "model-00006-of-00013.safetensors", + "gpt_neox.layers.11.mlp.dense_4h_to_h.weight": "model-00006-of-00013.safetensors", + "gpt_neox.layers.11.mlp.dense_h_to_4h.bias": "model-00006-of-00013.safetensors", + "gpt_neox.layers.11.mlp.dense_h_to_4h.weight": "model-00006-of-00013.safetensors", + "gpt_neox.layers.11.post_attention_layernorm.bias": "model-00005-of-00013.safetensors", + "gpt_neox.layers.11.post_attention_layernorm.weight": "model-00005-of-00013.safetensors", + "gpt_neox.layers.12.attention.bias": "model-00006-of-00013.safetensors", + "gpt_neox.layers.12.attention.dense.bias": "model-00006-of-00013.safetensors", + "gpt_neox.layers.12.attention.dense.weight": "model-00006-of-00013.safetensors", + "gpt_neox.layers.12.attention.masked_bias": "model-00006-of-00013.safetensors", + "gpt_neox.layers.12.attention.query_key_value.bias": "model-00006-of-00013.safetensors", + "gpt_neox.layers.12.attention.query_key_value.weight": "model-00006-of-00013.safetensors", + "gpt_neox.layers.12.attention.rotary_emb.inv_freq": "model-00006-of-00013.safetensors", + "gpt_neox.layers.12.input_layernorm.bias": "model-00006-of-00013.safetensors", + "gpt_neox.layers.12.input_layernorm.weight": "model-00006-of-00013.safetensors", + "gpt_neox.layers.12.mlp.dense_4h_to_h.bias": "model-00006-of-00013.safetensors", + "gpt_neox.layers.12.mlp.dense_4h_to_h.weight": "model-00006-of-00013.safetensors", + "gpt_neox.layers.12.mlp.dense_h_to_4h.bias": "model-00006-of-00013.safetensors", + "gpt_neox.layers.12.mlp.dense_h_to_4h.weight": "model-00006-of-00013.safetensors", + "gpt_neox.layers.12.post_attention_layernorm.bias": "model-00006-of-00013.safetensors", + "gpt_neox.layers.12.post_attention_layernorm.weight": "model-00006-of-00013.safetensors", + "gpt_neox.layers.13.attention.bias": "model-00006-of-00013.safetensors", + "gpt_neox.layers.13.attention.dense.bias": "model-00006-of-00013.safetensors", + "gpt_neox.layers.13.attention.dense.weight": "model-00006-of-00013.safetensors", + "gpt_neox.layers.13.attention.masked_bias": "model-00006-of-00013.safetensors", + "gpt_neox.layers.13.attention.query_key_value.bias": "model-00006-of-00013.safetensors", + "gpt_neox.layers.13.attention.query_key_value.weight": "model-00006-of-00013.safetensors", + "gpt_neox.layers.13.attention.rotary_emb.inv_freq": "model-00006-of-00013.safetensors", + "gpt_neox.layers.13.input_layernorm.bias": "model-00006-of-00013.safetensors", + "gpt_neox.layers.13.input_layernorm.weight": "model-00006-of-00013.safetensors", + "gpt_neox.layers.13.mlp.dense_4h_to_h.bias": "model-00007-of-00013.safetensors", + "gpt_neox.layers.13.mlp.dense_4h_to_h.weight": "model-00007-of-00013.safetensors", + "gpt_neox.layers.13.mlp.dense_h_to_4h.bias": "model-00007-of-00013.safetensors", + "gpt_neox.layers.13.mlp.dense_h_to_4h.weight": "model-00007-of-00013.safetensors", + "gpt_neox.layers.13.post_attention_layernorm.bias": "model-00006-of-00013.safetensors", + "gpt_neox.layers.13.post_attention_layernorm.weight": "model-00006-of-00013.safetensors", + "gpt_neox.layers.14.attention.bias": "model-00007-of-00013.safetensors", + "gpt_neox.layers.14.attention.dense.bias": "model-00007-of-00013.safetensors", + "gpt_neox.layers.14.attention.dense.weight": "model-00007-of-00013.safetensors", + "gpt_neox.layers.14.attention.masked_bias": "model-00007-of-00013.safetensors", + "gpt_neox.layers.14.attention.query_key_value.bias": "model-00007-of-00013.safetensors", + "gpt_neox.layers.14.attention.query_key_value.weight": "model-00007-of-00013.safetensors", + "gpt_neox.layers.14.attention.rotary_emb.inv_freq": "model-00007-of-00013.safetensors", + "gpt_neox.layers.14.input_layernorm.bias": "model-00007-of-00013.safetensors", + "gpt_neox.layers.14.input_layernorm.weight": "model-00007-of-00013.safetensors", + "gpt_neox.layers.14.mlp.dense_4h_to_h.bias": "model-00007-of-00013.safetensors", + "gpt_neox.layers.14.mlp.dense_4h_to_h.weight": "model-00007-of-00013.safetensors", + "gpt_neox.layers.14.mlp.dense_h_to_4h.bias": "model-00007-of-00013.safetensors", + "gpt_neox.layers.14.mlp.dense_h_to_4h.weight": "model-00007-of-00013.safetensors", + "gpt_neox.layers.14.post_attention_layernorm.bias": "model-00007-of-00013.safetensors", + "gpt_neox.layers.14.post_attention_layernorm.weight": "model-00007-of-00013.safetensors", + "gpt_neox.layers.15.attention.bias": "model-00007-of-00013.safetensors", + "gpt_neox.layers.15.attention.dense.bias": "model-00007-of-00013.safetensors", + "gpt_neox.layers.15.attention.dense.weight": "model-00007-of-00013.safetensors", + "gpt_neox.layers.15.attention.masked_bias": "model-00007-of-00013.safetensors", + "gpt_neox.layers.15.attention.query_key_value.bias": "model-00007-of-00013.safetensors", + "gpt_neox.layers.15.attention.query_key_value.weight": "model-00007-of-00013.safetensors", + "gpt_neox.layers.15.attention.rotary_emb.inv_freq": "model-00007-of-00013.safetensors", + "gpt_neox.layers.15.input_layernorm.bias": "model-00007-of-00013.safetensors", + "gpt_neox.layers.15.input_layernorm.weight": "model-00007-of-00013.safetensors", + "gpt_neox.layers.15.mlp.dense_4h_to_h.bias": "model-00008-of-00013.safetensors", + "gpt_neox.layers.15.mlp.dense_4h_to_h.weight": "model-00008-of-00013.safetensors", + "gpt_neox.layers.15.mlp.dense_h_to_4h.bias": "model-00007-of-00013.safetensors", + "gpt_neox.layers.15.mlp.dense_h_to_4h.weight": "model-00007-of-00013.safetensors", + "gpt_neox.layers.15.post_attention_layernorm.bias": "model-00007-of-00013.safetensors", + "gpt_neox.layers.15.post_attention_layernorm.weight": "model-00007-of-00013.safetensors", + "gpt_neox.layers.16.attention.bias": "model-00008-of-00013.safetensors", + "gpt_neox.layers.16.attention.dense.bias": "model-00008-of-00013.safetensors", + "gpt_neox.layers.16.attention.dense.weight": "model-00008-of-00013.safetensors", + "gpt_neox.layers.16.attention.masked_bias": "model-00008-of-00013.safetensors", + "gpt_neox.layers.16.attention.query_key_value.bias": "model-00008-of-00013.safetensors", + "gpt_neox.layers.16.attention.query_key_value.weight": "model-00008-of-00013.safetensors", + "gpt_neox.layers.16.attention.rotary_emb.inv_freq": "model-00008-of-00013.safetensors", + "gpt_neox.layers.16.input_layernorm.bias": "model-00008-of-00013.safetensors", + "gpt_neox.layers.16.input_layernorm.weight": "model-00008-of-00013.safetensors", + "gpt_neox.layers.16.mlp.dense_4h_to_h.bias": "model-00008-of-00013.safetensors", + "gpt_neox.layers.16.mlp.dense_4h_to_h.weight": "model-00008-of-00013.safetensors", + "gpt_neox.layers.16.mlp.dense_h_to_4h.bias": "model-00008-of-00013.safetensors", + "gpt_neox.layers.16.mlp.dense_h_to_4h.weight": "model-00008-of-00013.safetensors", + "gpt_neox.layers.16.post_attention_layernorm.bias": "model-00008-of-00013.safetensors", + "gpt_neox.layers.16.post_attention_layernorm.weight": "model-00008-of-00013.safetensors", + "gpt_neox.layers.17.attention.bias": "model-00008-of-00013.safetensors", + "gpt_neox.layers.17.attention.dense.bias": "model-00008-of-00013.safetensors", + "gpt_neox.layers.17.attention.dense.weight": "model-00008-of-00013.safetensors", + "gpt_neox.layers.17.attention.masked_bias": "model-00008-of-00013.safetensors", + "gpt_neox.layers.17.attention.query_key_value.bias": "model-00008-of-00013.safetensors", + "gpt_neox.layers.17.attention.query_key_value.weight": "model-00008-of-00013.safetensors", + "gpt_neox.layers.17.attention.rotary_emb.inv_freq": "model-00008-of-00013.safetensors", + "gpt_neox.layers.17.input_layernorm.bias": "model-00008-of-00013.safetensors", + "gpt_neox.layers.17.input_layernorm.weight": "model-00008-of-00013.safetensors", + "gpt_neox.layers.17.mlp.dense_4h_to_h.bias": "model-00008-of-00013.safetensors", + "gpt_neox.layers.17.mlp.dense_4h_to_h.weight": "model-00008-of-00013.safetensors", + "gpt_neox.layers.17.mlp.dense_h_to_4h.bias": "model-00008-of-00013.safetensors", + "gpt_neox.layers.17.mlp.dense_h_to_4h.weight": "model-00008-of-00013.safetensors", + "gpt_neox.layers.17.post_attention_layernorm.bias": "model-00008-of-00013.safetensors", + "gpt_neox.layers.17.post_attention_layernorm.weight": "model-00008-of-00013.safetensors", + "gpt_neox.layers.18.attention.bias": "model-00008-of-00013.safetensors", + "gpt_neox.layers.18.attention.dense.bias": "model-00009-of-00013.safetensors", + "gpt_neox.layers.18.attention.dense.weight": "model-00009-of-00013.safetensors", + "gpt_neox.layers.18.attention.masked_bias": "model-00008-of-00013.safetensors", + "gpt_neox.layers.18.attention.query_key_value.bias": "model-00009-of-00013.safetensors", + "gpt_neox.layers.18.attention.query_key_value.weight": "model-00009-of-00013.safetensors", + "gpt_neox.layers.18.attention.rotary_emb.inv_freq": "model-00008-of-00013.safetensors", + "gpt_neox.layers.18.input_layernorm.bias": "model-00008-of-00013.safetensors", + "gpt_neox.layers.18.input_layernorm.weight": "model-00008-of-00013.safetensors", + "gpt_neox.layers.18.mlp.dense_4h_to_h.bias": "model-00009-of-00013.safetensors", + "gpt_neox.layers.18.mlp.dense_4h_to_h.weight": "model-00009-of-00013.safetensors", + "gpt_neox.layers.18.mlp.dense_h_to_4h.bias": "model-00009-of-00013.safetensors", + "gpt_neox.layers.18.mlp.dense_h_to_4h.weight": "model-00009-of-00013.safetensors", + "gpt_neox.layers.18.post_attention_layernorm.bias": "model-00008-of-00013.safetensors", + "gpt_neox.layers.18.post_attention_layernorm.weight": "model-00008-of-00013.safetensors", + "gpt_neox.layers.19.attention.bias": "model-00009-of-00013.safetensors", + "gpt_neox.layers.19.attention.dense.bias": "model-00009-of-00013.safetensors", + "gpt_neox.layers.19.attention.dense.weight": "model-00009-of-00013.safetensors", + "gpt_neox.layers.19.attention.masked_bias": "model-00009-of-00013.safetensors", + "gpt_neox.layers.19.attention.query_key_value.bias": "model-00009-of-00013.safetensors", + "gpt_neox.layers.19.attention.query_key_value.weight": "model-00009-of-00013.safetensors", + "gpt_neox.layers.19.attention.rotary_emb.inv_freq": "model-00009-of-00013.safetensors", + "gpt_neox.layers.19.input_layernorm.bias": "model-00009-of-00013.safetensors", + "gpt_neox.layers.19.input_layernorm.weight": "model-00009-of-00013.safetensors", + "gpt_neox.layers.19.mlp.dense_4h_to_h.bias": "model-00009-of-00013.safetensors", + "gpt_neox.layers.19.mlp.dense_4h_to_h.weight": "model-00009-of-00013.safetensors", + "gpt_neox.layers.19.mlp.dense_h_to_4h.bias": "model-00009-of-00013.safetensors", + "gpt_neox.layers.19.mlp.dense_h_to_4h.weight": "model-00009-of-00013.safetensors", + "gpt_neox.layers.19.post_attention_layernorm.bias": "model-00009-of-00013.safetensors", + "gpt_neox.layers.19.post_attention_layernorm.weight": "model-00009-of-00013.safetensors", + "gpt_neox.layers.2.attention.bias": "model-00002-of-00013.safetensors", + "gpt_neox.layers.2.attention.dense.bias": "model-00002-of-00013.safetensors", + "gpt_neox.layers.2.attention.dense.weight": "model-00002-of-00013.safetensors", + "gpt_neox.layers.2.attention.masked_bias": "model-00002-of-00013.safetensors", + "gpt_neox.layers.2.attention.query_key_value.bias": "model-00002-of-00013.safetensors", + "gpt_neox.layers.2.attention.query_key_value.weight": "model-00002-of-00013.safetensors", + "gpt_neox.layers.2.attention.rotary_emb.inv_freq": "model-00002-of-00013.safetensors", + "gpt_neox.layers.2.input_layernorm.bias": "model-00002-of-00013.safetensors", + "gpt_neox.layers.2.input_layernorm.weight": "model-00002-of-00013.safetensors", + "gpt_neox.layers.2.mlp.dense_4h_to_h.bias": "model-00002-of-00013.safetensors", + "gpt_neox.layers.2.mlp.dense_4h_to_h.weight": "model-00002-of-00013.safetensors", + "gpt_neox.layers.2.mlp.dense_h_to_4h.bias": "model-00002-of-00013.safetensors", + "gpt_neox.layers.2.mlp.dense_h_to_4h.weight": "model-00002-of-00013.safetensors", + "gpt_neox.layers.2.post_attention_layernorm.bias": "model-00002-of-00013.safetensors", + "gpt_neox.layers.2.post_attention_layernorm.weight": "model-00002-of-00013.safetensors", + "gpt_neox.layers.20.attention.bias": "model-00009-of-00013.safetensors", + "gpt_neox.layers.20.attention.dense.bias": "model-00009-of-00013.safetensors", + "gpt_neox.layers.20.attention.dense.weight": "model-00009-of-00013.safetensors", + "gpt_neox.layers.20.attention.masked_bias": "model-00009-of-00013.safetensors", + "gpt_neox.layers.20.attention.query_key_value.bias": "model-00009-of-00013.safetensors", + "gpt_neox.layers.20.attention.query_key_value.weight": "model-00009-of-00013.safetensors", + "gpt_neox.layers.20.attention.rotary_emb.inv_freq": "model-00009-of-00013.safetensors", + "gpt_neox.layers.20.input_layernorm.bias": "model-00009-of-00013.safetensors", + "gpt_neox.layers.20.input_layernorm.weight": "model-00009-of-00013.safetensors", + "gpt_neox.layers.20.mlp.dense_4h_to_h.bias": "model-00010-of-00013.safetensors", + "gpt_neox.layers.20.mlp.dense_4h_to_h.weight": "model-00010-of-00013.safetensors", + "gpt_neox.layers.20.mlp.dense_h_to_4h.bias": "model-00010-of-00013.safetensors", + "gpt_neox.layers.20.mlp.dense_h_to_4h.weight": "model-00010-of-00013.safetensors", + "gpt_neox.layers.20.post_attention_layernorm.bias": "model-00009-of-00013.safetensors", + "gpt_neox.layers.20.post_attention_layernorm.weight": "model-00009-of-00013.safetensors", + "gpt_neox.layers.21.attention.bias": "model-00010-of-00013.safetensors", + "gpt_neox.layers.21.attention.dense.bias": "model-00010-of-00013.safetensors", + "gpt_neox.layers.21.attention.dense.weight": "model-00010-of-00013.safetensors", + "gpt_neox.layers.21.attention.masked_bias": "model-00010-of-00013.safetensors", + "gpt_neox.layers.21.attention.query_key_value.bias": "model-00010-of-00013.safetensors", + "gpt_neox.layers.21.attention.query_key_value.weight": "model-00010-of-00013.safetensors", + "gpt_neox.layers.21.attention.rotary_emb.inv_freq": "model-00010-of-00013.safetensors", + "gpt_neox.layers.21.input_layernorm.bias": "model-00010-of-00013.safetensors", + "gpt_neox.layers.21.input_layernorm.weight": "model-00010-of-00013.safetensors", + "gpt_neox.layers.21.mlp.dense_4h_to_h.bias": "model-00010-of-00013.safetensors", + "gpt_neox.layers.21.mlp.dense_4h_to_h.weight": "model-00010-of-00013.safetensors", + "gpt_neox.layers.21.mlp.dense_h_to_4h.bias": "model-00010-of-00013.safetensors", + "gpt_neox.layers.21.mlp.dense_h_to_4h.weight": "model-00010-of-00013.safetensors", + "gpt_neox.layers.21.post_attention_layernorm.bias": "model-00010-of-00013.safetensors", + "gpt_neox.layers.21.post_attention_layernorm.weight": "model-00010-of-00013.safetensors", + "gpt_neox.layers.22.attention.bias": "model-00010-of-00013.safetensors", + "gpt_neox.layers.22.attention.dense.bias": "model-00010-of-00013.safetensors", + "gpt_neox.layers.22.attention.dense.weight": "model-00010-of-00013.safetensors", + "gpt_neox.layers.22.attention.masked_bias": "model-00010-of-00013.safetensors", + "gpt_neox.layers.22.attention.query_key_value.bias": "model-00010-of-00013.safetensors", + "gpt_neox.layers.22.attention.query_key_value.weight": "model-00010-of-00013.safetensors", + "gpt_neox.layers.22.attention.rotary_emb.inv_freq": "model-00010-of-00013.safetensors", + "gpt_neox.layers.22.input_layernorm.bias": "model-00010-of-00013.safetensors", + "gpt_neox.layers.22.input_layernorm.weight": "model-00010-of-00013.safetensors", + "gpt_neox.layers.22.mlp.dense_4h_to_h.bias": "model-00011-of-00013.safetensors", + "gpt_neox.layers.22.mlp.dense_4h_to_h.weight": "model-00011-of-00013.safetensors", + "gpt_neox.layers.22.mlp.dense_h_to_4h.bias": "model-00010-of-00013.safetensors", + "gpt_neox.layers.22.mlp.dense_h_to_4h.weight": "model-00010-of-00013.safetensors", + "gpt_neox.layers.22.post_attention_layernorm.bias": "model-00010-of-00013.safetensors", + "gpt_neox.layers.22.post_attention_layernorm.weight": "model-00010-of-00013.safetensors", + "gpt_neox.layers.23.attention.bias": "model-00011-of-00013.safetensors", + "gpt_neox.layers.23.attention.dense.bias": "model-00011-of-00013.safetensors", + "gpt_neox.layers.23.attention.dense.weight": "model-00011-of-00013.safetensors", + "gpt_neox.layers.23.attention.masked_bias": "model-00011-of-00013.safetensors", + "gpt_neox.layers.23.attention.query_key_value.bias": "model-00011-of-00013.safetensors", + "gpt_neox.layers.23.attention.query_key_value.weight": "model-00011-of-00013.safetensors", + "gpt_neox.layers.23.attention.rotary_emb.inv_freq": "model-00011-of-00013.safetensors", + "gpt_neox.layers.23.input_layernorm.bias": "model-00011-of-00013.safetensors", + "gpt_neox.layers.23.input_layernorm.weight": "model-00011-of-00013.safetensors", + "gpt_neox.layers.23.mlp.dense_4h_to_h.bias": "model-00011-of-00013.safetensors", + "gpt_neox.layers.23.mlp.dense_4h_to_h.weight": "model-00011-of-00013.safetensors", + "gpt_neox.layers.23.mlp.dense_h_to_4h.bias": "model-00011-of-00013.safetensors", + "gpt_neox.layers.23.mlp.dense_h_to_4h.weight": "model-00011-of-00013.safetensors", + "gpt_neox.layers.23.post_attention_layernorm.bias": "model-00011-of-00013.safetensors", + "gpt_neox.layers.23.post_attention_layernorm.weight": "model-00011-of-00013.safetensors", + "gpt_neox.layers.24.attention.bias": "model-00011-of-00013.safetensors", + "gpt_neox.layers.24.attention.dense.bias": "model-00011-of-00013.safetensors", + "gpt_neox.layers.24.attention.dense.weight": "model-00011-of-00013.safetensors", + "gpt_neox.layers.24.attention.masked_bias": "model-00011-of-00013.safetensors", + "gpt_neox.layers.24.attention.query_key_value.bias": "model-00011-of-00013.safetensors", + "gpt_neox.layers.24.attention.query_key_value.weight": "model-00011-of-00013.safetensors", + "gpt_neox.layers.24.attention.rotary_emb.inv_freq": "model-00011-of-00013.safetensors", + "gpt_neox.layers.24.input_layernorm.bias": "model-00011-of-00013.safetensors", + "gpt_neox.layers.24.input_layernorm.weight": "model-00011-of-00013.safetensors", + "gpt_neox.layers.24.mlp.dense_4h_to_h.bias": "model-00011-of-00013.safetensors", + "gpt_neox.layers.24.mlp.dense_4h_to_h.weight": "model-00011-of-00013.safetensors", + "gpt_neox.layers.24.mlp.dense_h_to_4h.bias": "model-00011-of-00013.safetensors", + "gpt_neox.layers.24.mlp.dense_h_to_4h.weight": "model-00011-of-00013.safetensors", + "gpt_neox.layers.24.post_attention_layernorm.bias": "model-00011-of-00013.safetensors", + "gpt_neox.layers.24.post_attention_layernorm.weight": "model-00011-of-00013.safetensors", + "gpt_neox.layers.25.attention.bias": "model-00011-of-00013.safetensors", + "gpt_neox.layers.25.attention.dense.bias": "model-00012-of-00013.safetensors", + "gpt_neox.layers.25.attention.dense.weight": "model-00012-of-00013.safetensors", + "gpt_neox.layers.25.attention.masked_bias": "model-00011-of-00013.safetensors", + "gpt_neox.layers.25.attention.query_key_value.bias": "model-00012-of-00013.safetensors", + "gpt_neox.layers.25.attention.query_key_value.weight": "model-00012-of-00013.safetensors", + "gpt_neox.layers.25.attention.rotary_emb.inv_freq": "model-00011-of-00013.safetensors", + "gpt_neox.layers.25.input_layernorm.bias": "model-00011-of-00013.safetensors", + "gpt_neox.layers.25.input_layernorm.weight": "model-00011-of-00013.safetensors", + "gpt_neox.layers.25.mlp.dense_4h_to_h.bias": "model-00012-of-00013.safetensors", + "gpt_neox.layers.25.mlp.dense_4h_to_h.weight": "model-00012-of-00013.safetensors", + "gpt_neox.layers.25.mlp.dense_h_to_4h.bias": "model-00012-of-00013.safetensors", + "gpt_neox.layers.25.mlp.dense_h_to_4h.weight": "model-00012-of-00013.safetensors", + "gpt_neox.layers.25.post_attention_layernorm.bias": "model-00011-of-00013.safetensors", + "gpt_neox.layers.25.post_attention_layernorm.weight": "model-00011-of-00013.safetensors", + "gpt_neox.layers.26.attention.bias": "model-00012-of-00013.safetensors", + "gpt_neox.layers.26.attention.dense.bias": "model-00012-of-00013.safetensors", + "gpt_neox.layers.26.attention.dense.weight": "model-00012-of-00013.safetensors", + "gpt_neox.layers.26.attention.masked_bias": "model-00012-of-00013.safetensors", + "gpt_neox.layers.26.attention.query_key_value.bias": "model-00012-of-00013.safetensors", + "gpt_neox.layers.26.attention.query_key_value.weight": "model-00012-of-00013.safetensors", + "gpt_neox.layers.26.attention.rotary_emb.inv_freq": "model-00012-of-00013.safetensors", + "gpt_neox.layers.26.input_layernorm.bias": "model-00012-of-00013.safetensors", + "gpt_neox.layers.26.input_layernorm.weight": "model-00012-of-00013.safetensors", + "gpt_neox.layers.26.mlp.dense_4h_to_h.bias": "model-00012-of-00013.safetensors", + "gpt_neox.layers.26.mlp.dense_4h_to_h.weight": "model-00012-of-00013.safetensors", + "gpt_neox.layers.26.mlp.dense_h_to_4h.bias": "model-00012-of-00013.safetensors", + "gpt_neox.layers.26.mlp.dense_h_to_4h.weight": "model-00012-of-00013.safetensors", + "gpt_neox.layers.26.post_attention_layernorm.bias": "model-00012-of-00013.safetensors", + "gpt_neox.layers.26.post_attention_layernorm.weight": "model-00012-of-00013.safetensors", + "gpt_neox.layers.27.attention.bias": "model-00012-of-00013.safetensors", + "gpt_neox.layers.27.attention.dense.bias": "model-00012-of-00013.safetensors", + "gpt_neox.layers.27.attention.dense.weight": "model-00012-of-00013.safetensors", + "gpt_neox.layers.27.attention.masked_bias": "model-00012-of-00013.safetensors", + "gpt_neox.layers.27.attention.query_key_value.bias": "model-00012-of-00013.safetensors", + "gpt_neox.layers.27.attention.query_key_value.weight": "model-00012-of-00013.safetensors", + "gpt_neox.layers.27.attention.rotary_emb.inv_freq": "model-00012-of-00013.safetensors", + "gpt_neox.layers.27.input_layernorm.bias": "model-00012-of-00013.safetensors", + "gpt_neox.layers.27.input_layernorm.weight": "model-00012-of-00013.safetensors", + "gpt_neox.layers.27.mlp.dense_4h_to_h.bias": "model-00013-of-00013.safetensors", + "gpt_neox.layers.27.mlp.dense_4h_to_h.weight": "model-00013-of-00013.safetensors", + "gpt_neox.layers.27.mlp.dense_h_to_4h.bias": "model-00013-of-00013.safetensors", + "gpt_neox.layers.27.mlp.dense_h_to_4h.weight": "model-00013-of-00013.safetensors", + "gpt_neox.layers.27.post_attention_layernorm.bias": "model-00012-of-00013.safetensors", + "gpt_neox.layers.27.post_attention_layernorm.weight": "model-00012-of-00013.safetensors", + "gpt_neox.layers.3.attention.bias": "model-00002-of-00013.safetensors", + "gpt_neox.layers.3.attention.dense.bias": "model-00002-of-00013.safetensors", + "gpt_neox.layers.3.attention.dense.weight": "model-00002-of-00013.safetensors", + "gpt_neox.layers.3.attention.masked_bias": "model-00002-of-00013.safetensors", + "gpt_neox.layers.3.attention.query_key_value.bias": "model-00002-of-00013.safetensors", + "gpt_neox.layers.3.attention.query_key_value.weight": "model-00002-of-00013.safetensors", + "gpt_neox.layers.3.attention.rotary_emb.inv_freq": "model-00002-of-00013.safetensors", + "gpt_neox.layers.3.input_layernorm.bias": "model-00002-of-00013.safetensors", + "gpt_neox.layers.3.input_layernorm.weight": "model-00002-of-00013.safetensors", + "gpt_neox.layers.3.mlp.dense_4h_to_h.bias": "model-00002-of-00013.safetensors", + "gpt_neox.layers.3.mlp.dense_4h_to_h.weight": "model-00002-of-00013.safetensors", + "gpt_neox.layers.3.mlp.dense_h_to_4h.bias": "model-00002-of-00013.safetensors", + "gpt_neox.layers.3.mlp.dense_h_to_4h.weight": "model-00002-of-00013.safetensors", + "gpt_neox.layers.3.post_attention_layernorm.bias": "model-00002-of-00013.safetensors", + "gpt_neox.layers.3.post_attention_layernorm.weight": "model-00002-of-00013.safetensors", + "gpt_neox.layers.4.attention.bias": "model-00002-of-00013.safetensors", + "gpt_neox.layers.4.attention.dense.bias": "model-00003-of-00013.safetensors", + "gpt_neox.layers.4.attention.dense.weight": "model-00003-of-00013.safetensors", + "gpt_neox.layers.4.attention.masked_bias": "model-00002-of-00013.safetensors", + "gpt_neox.layers.4.attention.query_key_value.bias": "model-00003-of-00013.safetensors", + "gpt_neox.layers.4.attention.query_key_value.weight": "model-00003-of-00013.safetensors", + "gpt_neox.layers.4.attention.rotary_emb.inv_freq": "model-00002-of-00013.safetensors", + "gpt_neox.layers.4.input_layernorm.bias": "model-00002-of-00013.safetensors", + "gpt_neox.layers.4.input_layernorm.weight": "model-00002-of-00013.safetensors", + "gpt_neox.layers.4.mlp.dense_4h_to_h.bias": "model-00003-of-00013.safetensors", + "gpt_neox.layers.4.mlp.dense_4h_to_h.weight": "model-00003-of-00013.safetensors", + "gpt_neox.layers.4.mlp.dense_h_to_4h.bias": "model-00003-of-00013.safetensors", + "gpt_neox.layers.4.mlp.dense_h_to_4h.weight": "model-00003-of-00013.safetensors", + "gpt_neox.layers.4.post_attention_layernorm.bias": "model-00002-of-00013.safetensors", + "gpt_neox.layers.4.post_attention_layernorm.weight": "model-00002-of-00013.safetensors", + "gpt_neox.layers.5.attention.bias": "model-00003-of-00013.safetensors", + "gpt_neox.layers.5.attention.dense.bias": "model-00003-of-00013.safetensors", + "gpt_neox.layers.5.attention.dense.weight": "model-00003-of-00013.safetensors", + "gpt_neox.layers.5.attention.masked_bias": "model-00003-of-00013.safetensors", + "gpt_neox.layers.5.attention.query_key_value.bias": "model-00003-of-00013.safetensors", + "gpt_neox.layers.5.attention.query_key_value.weight": "model-00003-of-00013.safetensors", + "gpt_neox.layers.5.attention.rotary_emb.inv_freq": "model-00003-of-00013.safetensors", + "gpt_neox.layers.5.input_layernorm.bias": "model-00003-of-00013.safetensors", + "gpt_neox.layers.5.input_layernorm.weight": "model-00003-of-00013.safetensors", + "gpt_neox.layers.5.mlp.dense_4h_to_h.bias": "model-00003-of-00013.safetensors", + "gpt_neox.layers.5.mlp.dense_4h_to_h.weight": "model-00003-of-00013.safetensors", + "gpt_neox.layers.5.mlp.dense_h_to_4h.bias": "model-00003-of-00013.safetensors", + "gpt_neox.layers.5.mlp.dense_h_to_4h.weight": "model-00003-of-00013.safetensors", + "gpt_neox.layers.5.post_attention_layernorm.bias": "model-00003-of-00013.safetensors", + "gpt_neox.layers.5.post_attention_layernorm.weight": "model-00003-of-00013.safetensors", + "gpt_neox.layers.6.attention.bias": "model-00003-of-00013.safetensors", + "gpt_neox.layers.6.attention.dense.bias": "model-00003-of-00013.safetensors", + "gpt_neox.layers.6.attention.dense.weight": "model-00003-of-00013.safetensors", + "gpt_neox.layers.6.attention.masked_bias": "model-00003-of-00013.safetensors", + "gpt_neox.layers.6.attention.query_key_value.bias": "model-00003-of-00013.safetensors", + "gpt_neox.layers.6.attention.query_key_value.weight": "model-00003-of-00013.safetensors", + "gpt_neox.layers.6.attention.rotary_emb.inv_freq": "model-00003-of-00013.safetensors", + "gpt_neox.layers.6.input_layernorm.bias": "model-00003-of-00013.safetensors", + "gpt_neox.layers.6.input_layernorm.weight": "model-00003-of-00013.safetensors", + "gpt_neox.layers.6.mlp.dense_4h_to_h.bias": "model-00004-of-00013.safetensors", + "gpt_neox.layers.6.mlp.dense_4h_to_h.weight": "model-00004-of-00013.safetensors", + "gpt_neox.layers.6.mlp.dense_h_to_4h.bias": "model-00004-of-00013.safetensors", + "gpt_neox.layers.6.mlp.dense_h_to_4h.weight": "model-00004-of-00013.safetensors", + "gpt_neox.layers.6.post_attention_layernorm.bias": "model-00003-of-00013.safetensors", + "gpt_neox.layers.6.post_attention_layernorm.weight": "model-00003-of-00013.safetensors", + "gpt_neox.layers.7.attention.bias": "model-00004-of-00013.safetensors", + "gpt_neox.layers.7.attention.dense.bias": "model-00004-of-00013.safetensors", + "gpt_neox.layers.7.attention.dense.weight": "model-00004-of-00013.safetensors", + "gpt_neox.layers.7.attention.masked_bias": "model-00004-of-00013.safetensors", + "gpt_neox.layers.7.attention.query_key_value.bias": "model-00004-of-00013.safetensors", + "gpt_neox.layers.7.attention.query_key_value.weight": "model-00004-of-00013.safetensors", + "gpt_neox.layers.7.attention.rotary_emb.inv_freq": "model-00004-of-00013.safetensors", + "gpt_neox.layers.7.input_layernorm.bias": "model-00004-of-00013.safetensors", + "gpt_neox.layers.7.input_layernorm.weight": "model-00004-of-00013.safetensors", + "gpt_neox.layers.7.mlp.dense_4h_to_h.bias": "model-00004-of-00013.safetensors", + "gpt_neox.layers.7.mlp.dense_4h_to_h.weight": "model-00004-of-00013.safetensors", + "gpt_neox.layers.7.mlp.dense_h_to_4h.bias": "model-00004-of-00013.safetensors", + "gpt_neox.layers.7.mlp.dense_h_to_4h.weight": "model-00004-of-00013.safetensors", + "gpt_neox.layers.7.post_attention_layernorm.bias": "model-00004-of-00013.safetensors", + "gpt_neox.layers.7.post_attention_layernorm.weight": "model-00004-of-00013.safetensors", + "gpt_neox.layers.8.attention.bias": "model-00004-of-00013.safetensors", + "gpt_neox.layers.8.attention.dense.bias": "model-00004-of-00013.safetensors", + "gpt_neox.layers.8.attention.dense.weight": "model-00004-of-00013.safetensors", + "gpt_neox.layers.8.attention.masked_bias": "model-00004-of-00013.safetensors", + "gpt_neox.layers.8.attention.query_key_value.bias": "model-00004-of-00013.safetensors", + "gpt_neox.layers.8.attention.query_key_value.weight": "model-00004-of-00013.safetensors", + "gpt_neox.layers.8.attention.rotary_emb.inv_freq": "model-00004-of-00013.safetensors", + "gpt_neox.layers.8.input_layernorm.bias": "model-00004-of-00013.safetensors", + "gpt_neox.layers.8.input_layernorm.weight": "model-00004-of-00013.safetensors", + "gpt_neox.layers.8.mlp.dense_4h_to_h.bias": "model-00005-of-00013.safetensors", + "gpt_neox.layers.8.mlp.dense_4h_to_h.weight": "model-00005-of-00013.safetensors", + "gpt_neox.layers.8.mlp.dense_h_to_4h.bias": "model-00004-of-00013.safetensors", + "gpt_neox.layers.8.mlp.dense_h_to_4h.weight": "model-00004-of-00013.safetensors", + "gpt_neox.layers.8.post_attention_layernorm.bias": "model-00004-of-00013.safetensors", + "gpt_neox.layers.8.post_attention_layernorm.weight": "model-00004-of-00013.safetensors", + "gpt_neox.layers.9.attention.bias": "model-00005-of-00013.safetensors", + "gpt_neox.layers.9.attention.dense.bias": "model-00005-of-00013.safetensors", + "gpt_neox.layers.9.attention.dense.weight": "model-00005-of-00013.safetensors", + "gpt_neox.layers.9.attention.masked_bias": "model-00005-of-00013.safetensors", + "gpt_neox.layers.9.attention.query_key_value.bias": "model-00005-of-00013.safetensors", + "gpt_neox.layers.9.attention.query_key_value.weight": "model-00005-of-00013.safetensors", + "gpt_neox.layers.9.attention.rotary_emb.inv_freq": "model-00005-of-00013.safetensors", + "gpt_neox.layers.9.input_layernorm.bias": "model-00005-of-00013.safetensors", + "gpt_neox.layers.9.input_layernorm.weight": "model-00005-of-00013.safetensors", + "gpt_neox.layers.9.mlp.dense_4h_to_h.bias": "model-00005-of-00013.safetensors", + "gpt_neox.layers.9.mlp.dense_4h_to_h.weight": "model-00005-of-00013.safetensors", + "gpt_neox.layers.9.mlp.dense_h_to_4h.bias": "model-00005-of-00013.safetensors", + "gpt_neox.layers.9.mlp.dense_h_to_4h.weight": "model-00005-of-00013.safetensors", + "gpt_neox.layers.9.post_attention_layernorm.bias": "model-00005-of-00013.safetensors", + "gpt_neox.layers.9.post_attention_layernorm.weight": "model-00005-of-00013.safetensors" + } +} diff --git a/pytorch_model.bin b/pytorch_model.bin new file mode 100644 index 0000000..33e2fc2 --- /dev/null +++ b/pytorch_model.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d094ca7f73740d3dc7e161e1a8c3db944d08628b23306271b7c1f7a653c9b4bf +size 11887698501 diff --git a/special_tokens_map.json b/special_tokens_map.json new file mode 100644 index 0000000..b0b38c1 --- /dev/null +++ b/special_tokens_map.json @@ -0,0 +1,11 @@ +{ + "additional_special_tokens": [ + "<|endoftext|>", + "<|sep|>", + "<|acc|>", + "<|tel|>", + "<|rrn|>" + ], + "eos_token": "<|endoftext|>", + "pad_token": "<|endoftext|>" +} diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..379a60e --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:60e8bd123994badd563a603bce1a319dd7f5dc4798a476c2d88c671766db2f24 +size 1652158 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..3cf481a --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,6 @@ +{ + "name_or_path": "EleutherAI/polyglot-ko-5.8b", + "eos_token": "<|endoftext|>", + "pad_token": "<|endoftext|>", + "tokenizer_class": "PreTrainedTokenizerFast" +}