初始化项目，由ModelHub XC社区提供模型

Model: BUT-FIT/Czech-GPT-2-XL-133k
Source: Original Platform

.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,94 @@
+---
+language:
+- cs
+metrics:
+- perplexity
+pipeline_tag: text-generation
+license: mit
+datasets:
+- BUT-FIT/BUT-LCC
+---
+# Czech GPT
+This is our GPT-2 XL trained as a part of the research involved in [SemANT project](https://www.fit.vut.cz/research/project/1629/.en).
+
+# <span style="color:red">BUT LM Model Roster</span>
+- [BUT-FIT/CSTinyLlama-1.2B](https://huggingface.co/BUT-FIT/CSTinyLlama-1.2B)
+- [BUT-FIT/Czech-GPT-2-XL-133k](https://huggingface.co/BUT-FIT/Czech-GPT-2-XL-133k)
+- [BUT-FIT/csmpt7b](https://huggingface.co/BUT-FIT/csmpt7b)
+
+## Factsheet
+- The model is trained on our  `15,621,685,248 token/78,48 GB/10,900,000,000 word/18,800,000 paragraph` corpus of Czech obtained by Web Crawling.
+- The original size of our corpus before deduplication and lm-filtering steps was `266,44 GB`.
+- Our tokenizer size is 64k, and we use GPT-2 like BPE encoding for tokenization.
+- The model is trained in GPT-2 style, the first token is an actual text token (not bos). Thus first token probability can't be computed.
+- Due to the feature of our code, our model was never trained to generate [EOS]. 
+- The model was trained by 133,000 update steps (~139B training tokens), before the end of the experiment.
+- The model was adapted from the original GPT-2 XL, by: 
+   - replacing the tokenizer,  
+   - corresponding embeddings, and  
+   - copying over 1,000 EN representations corresponding to the 1,000 most frequent tokens into new embeddings based on a bilingual dictionary.
+- The training loss decreased steadily, and the model definitely didn't converge yet. We compare the loss to a small 124M model version.
+  <img src="XL_vs_SMALL_train.png" width="600"/>
+- The validation loss also decreased steadily. We had a bug in validation for early/late steps, so we released only validation from steps 46,000 to 100,000. Similarly, we compare the loss to the small 124M model version.
+  <img src="XL_vs_SMALL_test.png" width="600"/>
+
+## Training parameters
+Not mentioned parameters are the same as for GPT-2.
+
+| **Name**                   | **Value**     | **Note**                                                                                     |
+|----------------------------|---------------|----------------------------------------------------------------------------------------------|
+| dataset_type               | Concat        | Sequences at the model's input were concatenated up to `$max_seq_len`, divided by EOS token. |
+| tokenizer_size             | 64k           |                                                                                              |
+| max_seq_len                | 1024          |                                                                                              |
+| batch_size                 | 1024          |                                                                                              |
+| learning_rate              | 1.0e-4        |                                                                                              |
+| optimizer                  | LionW         |                                                                                              |
+| optimizer_betas            | 0.9/0.95      |                                                                                              |
+| optimizer_weight_decay     | 0             |                                                                                              |
+| optimizer_eps              | 1.0e-08       |                                                                                              |
+| gradient_clipping_max_norm | 1.0           |                                                                                              |
+| attn_impl                  | flash2        |                                                                                              |
+| dropout                    | 0.1           | for residuals, attention, embeddings                                                         |
+| fsdp                       | SHARD_GRAD_OP | (optimized for A100 40GB GPUs)                                                               |
+| precision                  | bf16          |                                                                                              |
+| scheduler                  | linear        |                                                                                              |
+| scheduler_warmup           | 10,000 steps  |                                                                                              |
+| scheduler_steps            | 200,000       |                                                                                              |
+| scheduler_alpha            | 0.1           | So LR on last step is 0.1*(vanilla LR)                                                       |
+
+## Usage
+```python
+from transformers import AutoTokenizer, AutoModelForCausalLM
+import torch
+
+t = AutoTokenizer.from_pretrained("BUT-FIT/Czech-GPT-2-XL-133k")
+m = AutoModelForCausalLM.from_pretrained("BUT-FIT/Czech-GPT-2-XL-133k").eval()
+
+# Try the model inference
+prompt = "Nejznámějším českým spisovatelem "
+input_ids = t.encode(prompt, return_tensors="pt")
+with torch.no_grad():
+    generated_text = m.generate(input_ids=input_ids,
+                                do_sample=True,
+                                top_p=0.95,
+                                repetition_penalty=1.0,
+                                temperature=0.8,
+                                max_new_tokens=64,
+                                num_return_sequences=1)
+    print(t.decode(generated_text[0], skip_special_tokens=True))
+```
+
+## Evaluation
+We observed 10-shot result improvement over the course of training for sentiment analysis, and hellaswag-like commonsense reasoning. 
+There were some tasks where there was no such improvement, such as grammar error classification (does the sentence contain grammatical error?).
+We will release the precise results once we advance with the work on our Czech evaluation kit.
+
+
+## Disclaimer
+This is an intermediate result of our work-in-progress. This is a probabilistic model, and authors are not responsible for the model outputs. Use at your own risk. 
+For further questions, turn to `martin.fajcik@vut.cz`.
+
+## Acknowledgement
+This work was supported by NAKI III program of  Ministry of Culture Czech Republic, project semANT --- 
+"Sémantický průzkumník textového kulturního dědictví" grant no. `DH23P03OVV060` and
+by the Ministry of Education, Youth and Sports of the Czech Republic through the e-INFRA CZ (ID:`90254`).

config.json

@@ -0,0 +1,41 @@
+{
+  "_name_or_path": "gpt2-xl",
+  "activation_function": "gelu_new",
+  "architectures": [
+    "GPT2LMHeadModel"
+  ],
+  "attn_impl": "flash2",
+  "attn_pdrop": 0.1,
+  "bos_token_id": null,
+  "embd_pdrop": 0.1,
+  "eos_token_id": 64001,
+  "initializer_range": 0.02,
+  "layer_norm_epsilon": 1e-05,
+  "model_type": "gpt2",
+  "n_ctx": 1024,
+  "n_embd": 1600,
+  "n_head": 25,
+  "n_inner": null,
+  "n_layer": 48,
+  "n_positions": 1024,
+  "output_past": true,
+  "reorder_and_upcast_attn": false,
+  "resid_pdrop": 0.1,
+  "scale_attn_by_inverse_layer_idx": false,
+  "scale_attn_weights": true,
+  "summary_activation": null,
+  "summary_first_dropout": 0.1,
+  "summary_proj_to_labels": true,
+  "summary_type": "cls_index",
+  "summary_use_proj": true,
+  "task_specific_params": {
+    "text-generation": {
+      "do_sample": true,
+      "max_length": 50
+    }
+  },
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.35.0.dev0",
+  "use_cache": true,
+  "vocab_size": 64002
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:806f849dec0163b0af73b03c855ab856680871a4ae9d6f8c552c121aca1392ab
+size 3464747832

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ddad4c8fac90602fd7d0cd10d2aa8a29a20ff5d2dbd7a3430ae73b7d4a271a18
+size 3464952161

special_tokens_map.json

@@ -0,0 +1,5 @@
+{
+  "eos_token": "[EOS]",
+  "pad_token": "[EOS]",
+  "unk_token": "[UNK]"
+}

tokenizer_config.json

@@ -0,0 +1,34 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "64000": {
+      "content": "[UNK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "64001": {
+      "content": "[EOS]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [],
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "[EOS]",
+  "model_max_length": 1024,
+  "tokenizer_class": "PreTrainedTokenizerFast",
+  "unk_token": "[UNK]"
+}