初始化项目，由ModelHub XC社区提供模型

Model: ai-forever/ruGPT-3.5-13B
Source: Original Platform

.gitattributes

@@ -0,0 +1,47 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bin.* filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zstandard filter=lfs diff=lfs merge=lfs -text
+*.tfevents* filter=lfs diff=lfs merge=lfs -text
+*.db* filter=lfs diff=lfs merge=lfs -text
+*.ark* filter=lfs diff=lfs merge=lfs -text
+**/*ckpt*data* filter=lfs diff=lfs merge=lfs -text
+**/*ckpt*.meta filter=lfs diff=lfs merge=lfs -text
+**/*ckpt*.index filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.gguf* filter=lfs diff=lfs merge=lfs -text
+*.ggml filter=lfs diff=lfs merge=lfs -text
+*.llamafile* filter=lfs diff=lfs merge=lfs -text
+*.pt2 filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,101 @@
+---
+license: mit
+language:
+- en
+- ru
+tags:
+- gpt3
+- transformers
+---
+# 🗿 ruGPT-3.5 13B
+
+Language model for Russian. Model has 13B parameters as you can guess from it's name. This is our biggest model so far and it was used for trainig GigaChat (read more about it in the [article](https://habr.com/ru/companies/sberbank/articles/730108/)).
+
+## Dataset
+
+Model was pretrained on a 300Gb of various domains, than additionaly trained on the 100 Gb of code and legal documets. Here is the dataset structure:
+
+![](https://habrastorage.org/getpro/habr/upload_files/384/cd1/40f/384cd140fbd9b4e7dd5427801be13ca0.png)
+
+Training data was deduplicated, the text deduplication includes 64-bit hashing of each text in the corpus for keeping texts with a unique hash. We also filter the documents based on their text compression rate using zlib4. The most strongly and weakly compressing deduplicated texts are discarded.
+
+## Technical details
+
+Model was trained using Deepspeed and Megatron libraries, on 300B tokens dataset for 3 epochs, around 45 days on 512 V100. After that model was finetuned 1 epoch with sequence length 2048 around 20 days on 200 GPU A100 on additional data (see above).
+
+After the final training perplexity for this model was around 8.8 for Russian.
+
+![](https://i.imgur.com/0yx67yl.png)
+
+## Examples of usage
+
+Try different generation strategies to reach better results.
+
+```python
+request = "Стих про программиста может быть таким:"
+
+encoded_input = tokenizer(request, return_tensors='pt', \
+                          add_special_tokens=False).to('cuda:0')
+output = model.generate(
+    **encoded_input,
+    num_beams=2,
+    do_sample=True,
+    max_new_tokens=100
+)
+
+print(tokenizer.decode(output[0], skip_special_tokens=True))
+```
+
+```
+>>> Стих про программиста может быть таким:
+
+    Программист сидит в кресле,
+    Стих сочиняет он про любовь,
+    Он пишет, пишет, пишет, пишет...
+    И не выходит ни черта!
+```
+
+```python
+request = "Нейронная сеть — это"
+
+encoded_input = tokenizer(request, return_tensors='pt', \
+                          add_special_tokens=False).to('cuda:0')
+output = model.generate(
+    **encoded_input,
+    num_beams=4,
+    do_sample=True,
+    max_new_tokens=100
+)
+
+print(tokenizer.decode(output[0], skip_special_tokens=True))
+```
+
+```
+>>> Нейронная сеть — это математическая модель, состоящая из большого
+    количества нейронов, соединенных между собой электрическими связями.
+    Нейронная сеть может быть смоделирована на компьютере, и с ее помощью
+    можно решать задачи, которые не поддаются решению с помощью традиционных
+    математических методов.
+```
+
+```python
+request = "Гагарин полетел в космос в"
+
+encoded_input = tokenizer(request, return_tensors='pt', \
+                          add_special_tokens=False).to('cuda:0')
+output = model.generate(
+    **encoded_input,
+    num_beams=2,
+    do_sample=True,
+    max_new_tokens=100
+)
+
+print(tokenizer.decode(output[0], skip_special_tokens=True))
+```
+
+```
+>>> Гагарин полетел в космос в 1961 году. Это было первое в истории
+    человечества космическое путешествие. Юрий Гагарин совершил его
+    на космическом корабле Восток-1. Корабль был запущен с космодрома
+    Байконур.
+```

config.json

@@ -0,0 +1,33 @@
+{
+  "activation_function": "gelu_new",
+  "architectures": [
+    "GPT2LMHeadModel"
+  ],
+  "attn_pdrop": 0.1,
+  "bos_token_id": 2,
+  "embd_pdrop": 0.1,
+  "eos_token_id": 3,
+  "initializer_range": 0.02,
+  "layer_norm_epsilon": 1e-05,
+  "model_type": "gpt2",
+  "n_ctx": 2048,
+  "n_embd": 5120,
+  "n_head": 40,
+  "n_inner": null,
+  "n_layer": 40,
+  "n_positions": 2048,
+  "pad_token_id": 0,
+  "reorder_and_upcast_attn": false,
+  "resid_pdrop": 0.1,
+  "scale_attn_by_inverse_layer_idx": false,
+  "scale_attn_weights": true,
+  "summary_activation": null,
+  "summary_first_dropout": 0.1,
+  "summary_proj_to_labels": true,
+  "summary_type": "cls_index",
+  "summary_use_proj": true,
+  "torch_dtype": "float32",
+  "transformers_version": "4.27.1",
+  "use_cache": true,
+  "vocab_size": 50272
+}

configuration.json

@@ -0,0 +1 @@
+{"framework": "pytorch", "task": "text-generation", "allow_remote": true}

generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 2,
+  "eos_token_id": 3,
+  "pad_token_id": 0,
+  "transformers_version": "4.27.1"
+}

pytorch_model-00001-of-00006.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5d20bcb9a28075f71dd23833b181ed2c5f5e487ba7f4dcf37a2a969fb602bbbe
+size 9915043053

pytorch_model-00002-of-00006.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a2247cc3ce22a37c63846d8d555f7a147bc583323bef598c0f2edf22457a0ab2
+size 9678360353

pytorch_model-00003-of-00006.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7f808bf83a8c18208d2dac9a29971f84bfea4b89c9bb5a632ca1c43a61dc6363
+size 9682535513

pytorch_model-00004-of-00006.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a04a8981a06494721724635a362f8f37bbba7e275b8879d7f3d272b5c9444675
+size 9997128229

pytorch_model-00005-of-00006.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8acaf6cb13e72aa4c63c2445893b9d57285d1955fec4c16b1a923b082e63bf73
+size 9787413665

pytorch_model-00006-of-00006.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eae3fe13782cadbe83a2f2d26d82c44b7ce112cde50ec629d52f0c1316de008b
+size 3550887252

pytorch_model.bin.index.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b387f7bbd52864315cdb97845e84a3d930a58600ee5bf70ab9a46ddeaa9e2f80
+size 42140

special_tokens_map.json

@@ -0,0 +1,31 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "mask_token": "<mask>",
+  "pad_token": {
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,43 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "bos_token": {
+    "__type": "AddedToken",
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "clean_up_tokenization_spaces": true,
+  "eos_token": {
+    "__type": "AddedToken",
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "errors": "replace",
+  "mask_token": "<mask>",
+  "model_max_length": 2048,
+  "pad_token": {
+    "__type": "AddedToken",
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "padding_side": "left",
+  "tokenizer_class": "GPT2Tokenizer",
+  "truncation_side": "left",
+  "unk_token": {
+    "__type": "AddedToken",
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
+}