初始化项目，由ModelHub XC社区提供模型

Model: rmihaylov/gpt2-medium-bg
Source: Original Platform

.gitattributes

@@ -0,0 +1,28 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bin.* filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zstandard filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,85 @@
+---
+inference: false
+language:
+- bg
+license: mit
+datasets:
+- oscar
+- chitanka
+- wikipedia
+tags:
+- torch
+---
+
+# GPT-2
+
+Pretrained model on Bulgarian language using a causal language modeling (CLM) objective. It was introduced in
+[this paper](https://d4mucfpksywv.cloudfront.net/better-language-models/language_models_are_unsupervised_multitask_learners.pdf)
+and first released at [this page](https://openai.com/blog/better-language-models/).
+
+## Model description
+
+This is the **MEDIUM** version.
+
+The training data is Bulgarian text from [OSCAR](https://oscar-corpus.com/post/oscar-2019/), [Chitanka](https://chitanka.info/) and [Wikipedia](https://bg.wikipedia.org/).
+
+## Intended uses & limitations
+
+You can use the raw model for: 
+- text generation
+- auto-complete
+- spelling correction
+
+Or fine-tune it to a downstream task.
+
+### How to use
+
+Here is how to use this model in PyTorch:
+
+```python
+>>> from transformers import AutoModel, AutoTokenizer
+>>>
+>>> model_id = "rmihaylov/gpt2-medium-bg"
+>>> tokenizer = AutoTokenizer.from_pretrained(model_id)
+>>> model = AutoModel.from_pretrained(model_id, trust_remote_code=True)
+>>>
+>>> input_ids = tokenizer.encode(
+>>>     "Здравей,", 
+>>>     add_special_tokens=False, 
+>>>     return_tensors='pt')
+>>>
+>>> output_ids = model.generate(
+>>>     input_ids, 
+>>>     do_sample=True, 
+>>>     max_length=50, 
+>>>     top_p=0.92, 
+>>>     pad_token_id=2,
+>>>     top_k=0)
+>>>
+>>> output = tokenizer.decode(output_ids[0])
+>>>
+>>> output = output.replace('<|endoftext|>', '\n\n\n')
+>>> output = output.replace('<|unknown|>', '')
+>>> output = output.replace('▁', ' ')
+>>> output = output.replace('<|n|>', '\n')
+>>>
+>>> print(output)
+
+Здравей, господин Фиш. — Добс забеляза как пребледня Ривера. 
+ — Не си тръгвайте още. Имам да ви задам няколко въпроса. 
+ — Благодаря, благодаря. — Фиш не изчака да му покаже, че е забелязал жеста й
+```
+
+### Limitations and bias
+
+As the openAI team themselves point out in their
+[model card](https://github.com/openai/gpt-2/blob/master/model_card.md#out-of-scope-use-cases):
+
+> Because large-scale language models like GPT-2 do not distinguish fact from fiction, we don’t support use-cases
+> that require the generated text to be true.
+>
+> Additionally, language models like GPT-2 reflect the biases inherent to the systems they were trained on, so we do
+> not recommend that they be deployed into systems that interact with humans > unless the deployers first carry out a
+> study of biases relevant to the intended use-case. We found no statistically significant difference in gender, race,
+> and religious bias probes between 774M and 1.5B, implying all versions of GPT-2 should be approached with similar
+> levels of caution around use cases that are sensitive to biases around human attributes.

added_tokens.json

@@ -0,0 +1 @@
+{"[MASK]": 50260, "[SEP]": 50258, "<pad>": 50259, "<unk>": 50261, "[CLS]": 50257}

config.json

@@ -0,0 +1,44 @@
+{
+  "_name_or_path": "/content/drive/MyDrive/ColabModels/GPT2_MEDIUM/pytorch_model/",
+  "activation_function": "gelu_new",
+  "architectures": [
+    "GPT2LMHeadModel"
+  ],
+  "attn_pdrop": 0.1,
+  "auto_map": {
+    "AutoModel": "modeling_gpt2.GPT2LMHeadModel"
+  },
+  "bos_token_id": 50256,
+  "embd_pdrop": 0.1,
+  "eos_token_id": 50256,
+  "initializer_range": 0.02,
+  "layer_norm_epsilon": 1e-05,
+  "model_type": "gpt2",
+  "n_ctx": 1024,
+  "n_embd": 1024,
+  "n_head": 16,
+  "n_inner": null,
+  "n_layer": 24,
+  "n_positions": 1024,
+  "n_special": 0,
+  "predict_special_tokens": true,
+  "reorder_and_upcast_attn": false,
+  "resid_pdrop": 0.1,
+  "scale_attn_by_inverse_layer_idx": false,
+  "scale_attn_weights": true,
+  "summary_activation": null,
+  "summary_first_dropout": 0.1,
+  "summary_proj_to_labels": true,
+  "summary_type": "cls_index",
+  "summary_use_proj": true,
+  "task_specific_params": {
+    "text-generation": {
+      "do_sample": true,
+      "max_length": 50
+    }
+  },
+  "torch_dtype": "float32",
+  "transformers_version": "4.18.0",
+  "use_cache": true,
+  "vocab_size": 50257
+}

modeling_gpt2.py

@@ -0,0 +1,32 @@
+from torch import nn
+
+from transformers import GPT2LMHeadModel as GPT2LMHeadModelBase
+from transformers.models.gpt2.modeling_gpt2 import GPT2Block as GPT2BlockBase
+
+
+class GPT2Block(GPT2BlockBase):
+    def forward(self, x, layer_past=None, 
+                attention_mask=None, head_mask=None, use_cache=False,
+                encoder_hidden_states=None, encoder_attention_mask=None, output_attentions=None):
+      
+        x = self.ln_1(x)
+        output_attn = self.attn(
+            x, layer_past=layer_past,
+            attention_mask=attention_mask,
+            head_mask=head_mask,
+            use_cache=use_cache)
+        
+        a = output_attn[0]
+        x = x + a
+
+        m = self.mlp(self.ln_2(x))
+        x = x + m
+
+        outputs = (x,) + output_attn[1:]
+        return outputs
+
+
+class GPT2LMHeadModel(GPT2LMHeadModelBase):
+    def __init__(self, config):
+        super().__init__(config)
+        self.transformer.h = nn.ModuleList([GPT2Block(config, layer_idx) for layer_idx in range(config.n_layer)])

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9ac587d8e665b3567fd1575458c3f6870a1b9b712221a56879633af7ea0b5013
+size 1444562137

special_tokens_map.json

@@ -0,0 +1 @@
+{"bos_token": "[CLS]", "eos_token": "[SEP]", "unk_token": "<unk>", "sep_token": "[SEP]", "pad_token": "<pad>", "cls_token": "[CLS]", "mask_token": "[MASK]"}

spiece.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b074bba94a7a59567fa70b3188dd7139746dad62d9c5c89580063ea806c06349
+size 1446766

tokenizer_config.json

@@ -0,0 +1 @@
+{"do_lower_case": false, "remove_space": true, "keep_accents": true, "bos_token": "[CLS]", "eos_token": "[SEP]", "unk_token": "<unk>", "sep_token": "[SEP]", "pad_token": "<pad>", "cls_token": "[CLS]", "mask_token": {"content": "[MASK]", "single_word": false, "lstrip": true, "rstrip": false, "normalized": false, "__type": "AddedToken"}, "sp_model_kwargs": {}, "special_tokens_map_file": "/content/drive/MyDrive/ColabModels/GPT2_MEDIUM/pytorch_model/special_tokens_map.json", "name_or_path": "/content/tokenizer/", "tokenizer_class": "AlbertTokenizer"}