初始化项目，由ModelHub XC社区提供模型
Model: MU-NLPC/CzeGPT-2 Source: Original Platform
This commit is contained in:
ModelHub XC
28
.gitattributes
vendored
Normal file
28
.gitattributes
vendored
Normal file
@@ -0,0 +1,28 @@
|
||||
*.7z filter=lfs diff=lfs merge=lfs -text
|
||||
*.arrow filter=lfs diff=lfs merge=lfs -text
|
||||
*.bin filter=lfs diff=lfs merge=lfs -text
|
||||
*.bin.* filter=lfs diff=lfs merge=lfs -text
|
||||
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
||||
*.ftz filter=lfs diff=lfs merge=lfs -text
|
||||
*.gz filter=lfs diff=lfs merge=lfs -text
|
||||
*.h5 filter=lfs diff=lfs merge=lfs -text
|
||||
*.joblib filter=lfs diff=lfs merge=lfs -text
|
||||
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
||||
*.model filter=lfs diff=lfs merge=lfs -text
|
||||
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
||||
*.onnx filter=lfs diff=lfs merge=lfs -text
|
||||
*.ot filter=lfs diff=lfs merge=lfs -text
|
||||
*.parquet filter=lfs diff=lfs merge=lfs -text
|
||||
*.pb filter=lfs diff=lfs merge=lfs -text
|
||||
*.pt filter=lfs diff=lfs merge=lfs -text
|
||||
*.pth filter=lfs diff=lfs merge=lfs -text
|
||||
*.rar filter=lfs diff=lfs merge=lfs -text
|
||||
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
||||
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
||||
*.tflite filter=lfs diff=lfs merge=lfs -text
|
||||
*.tgz filter=lfs diff=lfs merge=lfs -text
|
||||
*.xz filter=lfs diff=lfs merge=lfs -text
|
||||
*.zip filter=lfs diff=lfs merge=lfs -text
|
||||
*.zstandard filter=lfs diff=lfs merge=lfs -text
|
||||
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
||||
model.safetensors filter=lfs diff=lfs merge=lfs -text
|
||||
36
README.md
Normal file
36
README.md
Normal file
@@ -0,0 +1,36 @@
|
||||
---
|
||||
language: cs
|
||||
|
||||
license: cc-by-nc-sa-4.0
|
||||
datasets:
|
||||
- csTenTen17
|
||||
---
|
||||
|
||||
# CzeGPT-2
|
||||
CzeGPT-2 is a Czech version of GPT-2 language model by OpenAI with LM Head on top. The model has the same architectural dimensions as the GPT-2 small (12 layers, 12 heads, 1024 tokens on input/output, and embedding vectors with 768 dimensions) resulting in 124 M trainable parameters. It was trained on a 5 GB slice of cleaned csTenTen17 dataset.
|
||||
|
||||
The model is a good building block for any down-stream task requiring autoregressive text generation.
|
||||
|
||||
# Tokenizer
|
||||
Along, we also provide a tokenizer (vocab and merges) with vocab size of 50257 that was used during the pre-training phase. It is the byte-level BPE tokenizer used in the original paper and was trained on the whole 5 GB train set.
|
||||
|
||||
# Training results
|
||||
The model's perplexity on a 250 MB random slice of csTenTen17 dataset is **42.12**. This value is unfortunately not directly comparable to any other model, since there is no competition in Czech autoregressive models yet (and comparison with models for other languages is meaningless, because of different tokenization and test data).
|
||||
|
||||
# Running the predictions
|
||||
The repository includes a simple Jupyter Notebook that can help with the first steps when using the model.
|
||||
|
||||
## How to cite
|
||||
Hájek A. and Horák A. *CzeGPT-2 – Training New Model for Czech Generative Text Processing Evaluated with the Summarization Task*.
|
||||
IEEE Access, vol. 12, 34570–34581, Elsevier, 2024. https://doi.org/10.1109/ACCESS.2024.3371689
|
||||
|
||||
|
||||
@article{hajek_horak2024,
|
||||
author = "Adam Hájek and Aleš Horák",
|
||||
title = "CzeGPT-2 -- Training New Model for Czech Generative Text Processing Evaluated with the Summarization Task",
|
||||
journal= "IEEE Access",
|
||||
year = "2024",
|
||||
volume = "12",
|
||||
pages = "34570--34581",
|
||||
doi = "10.1109/ACCESS.2024.3371689",
|
||||
}
|
||||
36
config.json
Normal file
36
config.json
Normal file
@@ -0,0 +1,36 @@
|
||||
{
|
||||
"_name_or_path": "gpt2",
|
||||
"activation_function": "gelu_new",
|
||||
"architectures": [
|
||||
"GPT2LMHeadModel"
|
||||
],
|
||||
"attn_pdrop": 0.1,
|
||||
"bos_token_id": 50256,
|
||||
"embd_pdrop": 0.1,
|
||||
"eos_token_id": 50256,
|
||||
"gradient_checkpointing": false,
|
||||
"initializer_range": 0.02,
|
||||
"layer_norm_epsilon": 1e-05,
|
||||
"model_type": "gpt2",
|
||||
"n_ctx": 1024,
|
||||
"n_embd": 768,
|
||||
"n_head": 12,
|
||||
"n_inner": null,
|
||||
"n_layer": 12,
|
||||
"n_positions": 1024,
|
||||
"resid_pdrop": 0.1,
|
||||
"summary_activation": null,
|
||||
"summary_first_dropout": 0.1,
|
||||
"summary_proj_to_labels": true,
|
||||
"summary_type": "cls_index",
|
||||
"summary_use_proj": true,
|
||||
"task_specific_params": {
|
||||
"text-generation": {
|
||||
"do_sample": true,
|
||||
"max_length": 50
|
||||
}
|
||||
},
|
||||
"transformers_version": "4.5.1",
|
||||
"use_cache": true,
|
||||
"vocab_size": 50257
|
||||
}
|
||||
129
generate_text.ipynb
Normal file
129
generate_text.ipynb
Normal file
@@ -0,0 +1,129 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Text generation with CzeGPT-2\n",
|
||||
"Download all the files from the <a href=\"https://huggingface.co/MU-NLPC/CzeGPT-2/tree/main\">CzeGPT-2 Hugging Face repository</a> and place `vocab.json` and `merges.txt` into one folder and `pytorch_model.bin` and `config.json` into another. \n",
|
||||
"\n",
|
||||
"Provide path to these folders later in the code."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from transformers import GPT2TokenizerFast, GPT2LMHeadModel"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# load model\n",
|
||||
"model_path = \"path/to/folder/with/model/and/config/files\"\n",
|
||||
"\n",
|
||||
"model = GPT2LMHeadModel.from_pretrained(model_path)\n",
|
||||
"\n",
|
||||
"# put model into eval mode and on device\n",
|
||||
"device = 'cuda' # 'cpu' alternatively\n",
|
||||
"model.eval();\n",
|
||||
"model.to(device)\n",
|
||||
"\n",
|
||||
"# load tokenizer\n",
|
||||
"tokenizer_path = \"path/to/folder/with/vocab/and/merges/files\"\n",
|
||||
"\n",
|
||||
"tokenizer = GPT2TokenizerFast.from_pretrained(tokenizer_path, pad_token='<|endoftext|>')"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 10,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# input\n",
|
||||
"prompt = \"Začátek na který chcete navázat\"\n",
|
||||
"input_ids = tokenizer.encode(prompt, return_tensors='pt').to('cuda')"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 12,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
">> Generated text 1\n",
|
||||
"\n",
|
||||
"Začátek na který chcete navázat. Na začátku si zvolíte téma a po krátkém hledání odpovědí se rozhodnete co dál. Za pár sekund je k dispozici další volba pro pokračování hry / mise. Po kliknutí se dostanete do hlavního menu. V něm je pro vás ještě několik voleb. Můžete si vybrat z více možností. Můžete si nastavit název mise, čas začátku hry, čas spuštění kampaně (v závislosti na zemi, ve které se hra odehrává / v USA a Německu). Poté si zvolíte způsob, jakým se do hry zapojí váš kamarád. Toto je jedna z voleb, kterou máte zvolit. Můžete hrát i ve více lidech, což zvyšuje herní zážitek. Při výběru se můžete stát součástí týmu spolu s kamarádem nebo můžete hrát samostatně. Jakožto hráč jste si sami vybráni, můžete začít s tréninkem, pokud chcete tak se můžete stát\n",
|
||||
"\n",
|
||||
"---\n",
|
||||
">> Generated text 2\n",
|
||||
"\n",
|
||||
"Začátek na který chcete navázat byla nová postava z anime. Příběh je prostý - Colin, O 'Brien a jeho společníci se chtějí dostat na palubu Colina a to má za důsledek i smrt Colina a jeho přátel, kteří s ním souhlasí, dokud na tuto situaci nedojde... Do této kategorie spadají i tzv. superhrdina, který si říká Colin a také je na cestě na palubu lodi Colinova přítele Colina. Na první pohled vypadají tato jména zajímavě, ale po chvíli zkoumání zjistíte, že vlastně není co řešit.. Colin se v poslední době příliš nezabývá svojí minulostí, což je způsobeno především faktem, že se po jeho smrti stává Colin jeho přítelkyní, kvůli které se rozhodl, že bude žít s Alekim, což znamená, že se stal Anitou\n",
|
||||
"\n",
|
||||
"---\n",
|
||||
">> Generated text 3\n",
|
||||
"\n",
|
||||
"Začátek na který chcete navázat je změna na již zavedený obor a jeho zařazení do jiného oboru. Při hledání a volbě oboru si můžete vybrat ze základních, ale i doplňkových. Chcete -li se odlišit od konkurence, můžete si vybrat obor, který je vám bližší. Název obor: Stavební výroba - základní zaměření, případně doplňkové specializace. Pokud se nechcete vzdát specializovaného oboru, pak si můžete vybrat některou ze základních stavebních specializací. Pokud se rozhodnete pro práci v oboru, pak se zaměřte pouze na stavebnictví. Pokud budete chtít vydělávat peníze, tak si můžete vybrat specializaci stavebního zaměření. Pokud budete ve svém oboru zaměstnávat pracovníky, kteří vykonávají práce ve výkopech, nebo chcete vydělávat peníze, tak si můžete vybrat specializaci stavebního zaměření s názvem Stavební práce, popř. Stavební služby, protože se nejedná o stejné\n",
|
||||
"\n",
|
||||
"---\n",
|
||||
"CPU times: user 4.45 s, sys: 1.26 s, total: 5.71 s\n",
|
||||
"Wall time: 53.4 s\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"%%time\n",
|
||||
"max_length = 160\n",
|
||||
"min_length = 50\n",
|
||||
"sample_outputs = model.generate(input_ids, pad_token_id=0,\n",
|
||||
" do_sample=True, \n",
|
||||
" max_length=max_length, \n",
|
||||
" min_length=max_length,\n",
|
||||
" top_k=50,\n",
|
||||
" num_return_sequences=3)\n",
|
||||
"\n",
|
||||
"for i, sample_output in enumerate(sample_outputs):\n",
|
||||
" print(\">> Generated text {}\\n\\n{}\".format(i+1, tokenizer.decode(sample_output.tolist())))\n",
|
||||
" print('\\n---')"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.8.5"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 4
|
||||
}
|
||||
50001
merges.txt
Normal file
50001
merges.txt
Normal file
File diff suppressed because it is too large
Load Diff
3
model.safetensors
Normal file
3
model.safetensors
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:bee2d3b98881344631c5bb5ee0dd4abb9fe302a48e49c56be25a5af0cd5b4403
|
||||
size 510359640
|
||||
3
pytorch_model.bin
Normal file
3
pytorch_model.bin
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:e1d4e15ff644fcb525c91c651ea72aed937f213940d1177337ea934d74ddced4
|
||||
size 510408315
|
||||
1
vocab.json
Normal file
1
vocab.json
Normal file
File diff suppressed because one or more lines are too long
Reference in New Issue
Block a user