From b334c8665369d96eac4ff6a5cd267bfa536bce7b Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Fri, 14 Aug 2026 12:47:16 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: colorfulscoop/gpt2-small-ja Source: Original Platform --- .gitattributes | 17 ++++++++ README.md | 92 +++++++++++++++++++++++++++++++++++++++++ config.json | 40 ++++++++++++++++++ pytorch_model.bin | 3 ++ special_tokens_map.json | 1 + spiece.model | 3 ++ tf_model.h5 | 3 ++ tokenizer_config.json | 1 + 8 files changed, 160 insertions(+) create mode 100644 .gitattributes create mode 100644 README.md create mode 100644 config.json create mode 100644 pytorch_model.bin create mode 100644 special_tokens_map.json create mode 100644 spiece.model create mode 100644 tf_model.h5 create mode 100644 tokenizer_config.json diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..d699711 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,17 @@ +*.bin.* filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tar.gz filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..59e9327 --- /dev/null +++ b/README.md @@ -0,0 +1,92 @@ +--- +language: ja +datasets: wikipedia +widget: +- text: 統計的機械学習でのニューラルネットワーク +license: cc +--- + +# GPT-2 small Japanese model + +This repository contains a GPT2-small model trained on Japanese Wikipedia dataset. + +## Training data + +[Japanese Wikipedia](https://ja.wikipedia.org/wiki/Wikipedia:データベースダウンロード) dataset as of Aug20, 2021 released under [Creative Commons Attribution-ShareAlike 3.0](https://creativecommons.org/licenses/by-sa/3.0/) is used for both tokenizer and GPT-2 model. + +We splitted the dataset into three subsets - train, valid and test sets. Both tokenizer and model were trained on the train set. +Train set contains around 540M tokens. + +## Model description + +The model architecture is the same as GPT-2 small model (n_ctx: 1024, n_embd 768, n_head: 12, n_layer: 12) except for a vocabulary size. +The vocabulary size is set to 32,000 instead of an original size of 50,257. +`transformers.GPT2LMHeadModel` is used for training. + +## Tokenizer description + +[SentencePiece](https://github.com/google/sentencepiece) is used as a tokenizer for this model. + +We utilized 1,000,000 sentences from train set. +The vocabulary size was 32,000. +A `add_dummy_prefix` option was set to `True` because Japanese words are not separated by whitespaces. + +After training, the tokenizer model was imported as `transformers.BERTGenerationTokenizer` +because it supports SentencePiece models and it does not add any special tokens as default, +which is useful expecially for a text generation task. + +## Training + +The model was trained on the train set for 30 epochs with batch size 32. Each sample contained 1024 tokens. + +We utilized Adam optimizer. Learning rate was linearly increased from `0` to `1e-4` during the first 10,000 steps. +A clip norm was set to `1.0`. + +Test set perplexity of the trained model was 29.13. + +Please refer to [GitHub](https://github.com/colorfulscoop/gpt-ja) for more training details. + +## Usage + +First, install dependecies. + +```sh +$ pip install transformers==4.10.0 torch==1.8.1 sentencepiece==0.1.96 +``` + +Then use pipeline to generate sentences. + +```sh +>>> import transformers +>>> pipeline = transformers.pipeline("text-generation", "colorfulscoop/gpt2-small-ja") +>>> pipeline("統計的機械学習でのニューラルネットワーク", do_sample=True, top_p=0.95, top_k=50, num_return_sequences=3) +``` + +**Note:** The default model configuration `config.json` sets parameters for text generation with `do_sample=True`, `top_k=50`, `top_p=0.95`. +Please set these parameters when you need to use different parameters. + +## Versions + +We recommend to specify `revision` to load the model for reproducibility. + +| Revision | Date of Wikipedia dump | +| --- | --- | +| 20210820.1.0 | Aug 20, 2021 | +| 20210301.1.0 | March 1, 2021 | + +You can specify `revision` as follows. + +```py +# Example of pipeline +>>> transformers.pipeline("text-generation", "colorfulscoop/gpt2-small-ja", revision="20210820.1.0") +# Example of AutoModel +>>> transformers.AutoModel.from_pretrained("colorfulscoop/gpt2-small-ja", revision="20210820.1.0") +``` + +## License + +All the models included in this repository are licensed under [Creative Commons Attribution-ShareAlike 3.0](https://creativecommons.org/licenses/by-sa/3.0/). + +**Disclaimer:** The model potentially has possibility that it generates similar texts in the training data, texts not to be true, or biased texts. Use of the model is at your sole risk. Colorful Scoop makes no warranty or guarantee of any outputs from the model. Colorful Scoop is not liable for any trouble, loss, or damage arising from the model output. + +**Author:** Colorful Scoop diff --git a/config.json b/config.json new file mode 100644 index 0000000..df70c35 --- /dev/null +++ b/config.json @@ -0,0 +1,40 @@ +{ + "_name_or_path": "models/gpt2-small", + "activation_function": "gelu_new", + "architectures": [ + "GPT2LMHeadModel" + ], + "attn_pdrop": 0.1, + "bos_token_id": 2, + "cls_token_id": 4, + "embd_pdrop": 0.1, + "eos_token_id": 3, + "gradient_checkpointing": false, + "initializer_range": 0.02, + "layer_norm_epsilon": 1e-05, + "model_type": "gpt2", + "n_ctx": 1024, + "n_embd": 768, + "n_head": 12, + "n_inner": null, + "n_layer": 12, + "n_positions": 1024, + "pad_token_id": 0, + "resid_pdrop": 0.1, + "scale_attn_weights": true, + "sep_token_id": 5, + "summary_activation": null, + "summary_first_dropout": 0.1, + "summary_proj_to_labels": true, + "summary_type": "cls_index", + "summary_use_proj": true, + "tokenizer_class": "BertGenerationTokenizer", + "torch_dtype": "float32", + "transformers_version": "4.10.0", + "unk_token_id": 1, + "use_cache": true, + "vocab_size": 32000, + "top_k": 50, + "top_p": 0.95, + "do_sample": true +} diff --git a/pytorch_model.bin b/pytorch_model.bin new file mode 100644 index 0000000..80fd276 --- /dev/null +++ b/pytorch_model.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1e95f3fb022ae9e599aaacf7eb9b69cc2194b817da1d07dca8de6ec6127d33d4 +size 454320757 diff --git a/special_tokens_map.json b/special_tokens_map.json new file mode 100644 index 0000000..8d854db --- /dev/null +++ b/special_tokens_map.json @@ -0,0 +1 @@ +{"bos_token": "", "eos_token": "", "unk_token": "", "sep_token": "", "pad_token": "", "cls_token": ""} \ No newline at end of file diff --git a/spiece.model b/spiece.model new file mode 100644 index 0000000..9322d25 --- /dev/null +++ b/spiece.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ec01688f1fdf79d9596d099bfe2cd1ec8d0871e848660ca643907a3e4a5fb97f +size 802969 diff --git a/tf_model.h5 b/tf_model.h5 new file mode 100644 index 0000000..f807394 --- /dev/null +++ b/tf_model.h5 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:20683b6b0ee0f9c2e6b017085fa169ae4ea62af56b46ff17334e312e6b8b1849 +size 441849416 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..c43c6c8 --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1 @@ +{"bos_token": "", "eos_token": "", "unk_token": "", "pad_token": "", "sep_token": "", "sp_model_kwargs": {}, "cls_token": "", "special_tokens_map_file": "models/small-v2/special_tokens_map.json", "tokenizer_file": null, "name_or_path": "models/small-v2/", "tokenizer_class": "BertGenerationTokenizer"} \ No newline at end of file