commit 57b968894f01781012d75422269f5f9f8f7ff1f5 Author: ModelHub XC Date: Thu Jul 16 19:30:08 2026 +0800 初始化项目，由ModelHub XC社区提供模型 Model: rinna/japanese-gpt2-xsmall Source: Original Platform diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..bd3e1cc --- /dev/null +++ b/.gitattributes @@ -0,0 +1,52 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text + + +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text + +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text + +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zstandard filter=lfs diff=lfs merge=lfs -text +*.tfevents* filter=lfs diff=lfs merge=lfs -text +*.db* filter=lfs diff=lfs merge=lfs -text +*.ark* filter=lfs diff=lfs merge=lfs -text +**/*ckpt*data* filter=lfs diff=lfs merge=lfs -text +**/*ckpt*.meta filter=lfs diff=lfs merge=lfs -text +**/*ckpt*.index filter=lfs diff=lfs merge=lfs -text + +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.gguf* filter=lfs diff=lfs merge=lfs -text +*.ggml filter=lfs diff=lfs merge=lfs -text +*.llamafile* filter=lfs diff=lfs merge=lfs -text +*.pt2 filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text + +model.safetensors filter=lfs diff=lfs merge=lfs -text +tf_model.h5 filter=lfs diff=lfs merge=lfs -text +pytorch_model.bin filter=lfs diff=lfs merge=lfs -text +spiece.model filter=lfs diff=lfs merge=lfs -text \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..967f8fc --- /dev/null +++ b/README.md @@ -0,0 +1,67 @@ +--- +language: ja +thumbnail: https://github.com/rinnakk/japanese-gpt2/blob/master/rinna.png +tags: +- gpt2 +- text-generation +- lm +- nlp +license: mit +datasets: +- cc100 +- wikipedia +widget: +- text: "生命、宇宙、そして万物についての究極の疑問の答えは" +--- + +# japanese-gpt2-xsmall + +![rinna-icon](./rinna.png) + +This repository provides an extra-small-sized Japanese GPT-2 model. The model was trained using code from Github repository [rinnakk/japanese-pretrained-models](https://github.com/rinnakk/japanese-pretrained-models) by [rinna Co., Ltd.](https://corp.rinna.co.jp/) + +# How to use the model + +~~~~ +from transformers import AutoTokenizer, AutoModelForCausalLM + +tokenizer = AutoTokenizer.from_pretrained("rinna/japanese-gpt2-xsmall", use_fast=False) +tokenizer.do_lower_case = True # due to some bug of tokenizer config loading + +model = AutoModelForCausalLM.from_pretrained("rinna/japanese-gpt2-xsmall") +~~~~ + +# Model architecture +A 6-layer, 512-hidden-size transformer-based language model. + +# Training +The model was trained on [Japanese CC-100](http://data.statmt.org/cc-100/ja.txt.xz) and [Japanese Wikipedia](https://dumps.wikimedia.org/other/cirrussearch) to optimize a traditional language modelling objective on 8\\*V100 GPUs for around 4 days. It reaches around 28 perplexity on a chosen validation set from CC-100. + +# Tokenization +The model uses a [sentencepiece](https://github.com/google/sentencepiece)-based tokenizer, the vocabulary was trained on the Japanese Wikipedia using the official sentencepiece training script. + +# Release date +August 25, 2021 + +# How to cite +```bibtex +@misc{rinna-japanese-gpt2-xsmall, + title = {rinna/japanese-gpt2-xsmall}, + author = {Zhao, Tianyu and Sawada, Kei}, + url = {https://huggingface.co/rinna/japanese-gpt2-xsmall} +} + +@inproceedings{sawada2024release, + title = {Release of Pre-Trained Models for the {J}apanese Language}, + author = {Sawada, Kei and Zhao, Tianyu and Shing, Makoto and Mitsui, Kentaro and Kaga, Akio and Hono, Yukiya and Wakatsuki, Toshiaki and Mitsuda, Koh}, + booktitle = {Proceedings of the 2024 Joint International Conference on Computational Linguistics, Language Resources and Evaluation (LREC-COLING 2024)}, + month = {5}, + year = {2024}, + pages = {13898--13905}, + url = {https://aclanthology.org/2024.lrec-main.1213}, + note = {\url{https://arxiv.org/abs/2404.01657}} +} +``` + +# Licenese +[The MIT license](https://opensource.org/licenses/MIT) diff --git a/config.json b/config.json new file mode 100644 index 0000000..b51bb47 --- /dev/null +++ b/config.json @@ -0,0 +1,37 @@ +{ + "_name_or_path": "rinna/japanese-gpt2-xsmall", + "activation_function": "gelu_new", + "architectures": [ + "GPT2LMHeadModel" + ], + "attn_pdrop": 0.1, + "bos_token_id": 1, + "embd_pdrop": 0.1, + "eos_token_id": 2, + "gradient_checkpointing": false, + "initializer_range": 0.02, + "layer_norm_epsilon": 1e-05, + "model_type": "gpt2", + "n_ctx": 1024, + "n_embd": 512, + "n_head": 8, + "n_inner": 2304, + "n_layer": 6, + "n_positions": 1024, + "resid_pdrop": 0.1, + "scale_attn_weights": true, + "summary_activation": null, + "summary_first_dropout": 0.1, + "summary_proj_to_labels": true, + "summary_type": "cls_index", + "summary_use_proj": true, + "task_specific_params": { + "text-generation": { + "do_sample": true, + "max_length": 50 + } + }, + "transformers_version": "4.8.2", + "use_cache": true, + "vocab_size": 32000 +} diff --git a/configuration.json b/configuration.json new file mode 100644 index 0000000..bbeeda1 --- /dev/null +++ b/configuration.json @@ -0,0 +1 @@ +{"framework": "pytorch", "task": "text-generation", "allow_remote": true} \ No newline at end of file diff --git a/model.safetensors b/model.safetensors new file mode 100644 index 0000000..ebf2f10 --- /dev/null +++ b/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:06662948dfa3227e3a42200787f28e86657d0f1e37a4ba1439452419581a4229 +size 155892312 diff --git a/pytorch_model.bin b/pytorch_model.bin new file mode 100644 index 0000000..0e914c0 --- /dev/null +++ b/pytorch_model.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:390f67d37f087edfc91e506d7540afb3726af90b0d68e4e8f312f826aada560b +size 155916167 diff --git a/rinna.png b/rinna.png new file mode 100644 index 0000000..d0f007e Binary files /dev/null and b/rinna.png differ diff --git a/special_tokens_map.json b/special_tokens_map.json new file mode 100644 index 0000000..6d5b058 --- /dev/null +++ b/special_tokens_map.json @@ -0,0 +1 @@ +{"bos_token": "", "eos_token": "", "unk_token": "", "sep_token": "[SEP]", "pad_token": "[PAD]", "cls_token": "[CLS]", "mask_token": "[MASK]"} \ No newline at end of file diff --git a/spiece.model b/spiece.model new file mode 100644 index 0000000..a14f72b --- /dev/null +++ b/spiece.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b5cbdfa8aa7c54c8c5af85b78c309c54a5f2749a20468bf6f60eee007fe6fec1 +size 805634 diff --git a/tf_model.h5 b/tf_model.h5 new file mode 100644 index 0000000..852361c --- /dev/null +++ b/tf_model.h5 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:41f1f2f973544887a788ad1d034331ee35e728aa1963b1dc1d8bd082ce27bb35 +size 149686584 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..456c3d2 --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1 @@ +{"eos_token": "", "unk_token": "", "pad_token": "[PAD]", "extra_ids": 0, "additional_special_tokens": [], "sp_model_kwargs": {}, "bos_token": "", "cls_token": "[CLS]", "sep_token": "[SEP]", "mask_token": "[MASK]", "do_lower_case": true, "tokenizer_class": "T5Tokenizer"} \ No newline at end of file