From 5e9df6ca208af74f300fa6e516641b803d47872c Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Fri, 31 Jul 2026 22:11:17 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: abeja/gpt2-large-japanese Source: Original Platform --- .gitattributes | 31 +++++++++++ README.md | 113 ++++++++++++++++++++++++++++++++++++++++ config.json | 40 ++++++++++++++ pytorch_model.bin | 3 ++ special_tokens_map.json | 1 + spiece.model | 3 ++ tf_model.h5 | 3 ++ tokenizer_config.json | 1 + 8 files changed, 195 insertions(+) create mode 100644 .gitattributes create mode 100644 README.md create mode 100644 config.json create mode 100644 pytorch_model.bin create mode 100644 special_tokens_map.json create mode 100644 spiece.model create mode 100644 tf_model.h5 create mode 100644 tokenizer_config.json diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..78be843 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,31 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..5d28882 --- /dev/null +++ b/README.md @@ -0,0 +1,113 @@ +--- +language: ja +tags: +- ja +- japanese +- gpt2 +- text-generation +- lm +- nlp +license: mit +datasets: +- cc100 +- wikipedia +- oscar +widget: +- text: "人とAIが協調するためには、" +--- + +# gpt2-large-japanese + +This repository provides a large sized Japanese GPT-2 model. The model was trained by [ABEJA, Inc](https://abejainc.com/) + +# How to use +First, install sentencepiece. We have confirmed behavior with the latest version August 2022. (Skip if not necessary.) + +``` shell +pip install sentencepiece +``` + +When using pipeline for text generation. + +``` python +from transformers import pipeline + + +generator = pipeline("text-generation", model="abeja/gpt2-large-japanese") +generated = generator( + "人とAIが協調するためには、", + max_length=30, + do_sample=True, + num_return_sequences=3, + top_p=0.95, + top_k=50, + pad_token_id=3 +) +print(*generated, sep="\n") + +""" +[out] +{'generated_text': '人とAIが協調するためには、社会的なルールをきちんと理解して、人と共存し、協働して生きていくのが重要だという。'} +{'generated_text': '人とAIが協調するためには、それぞれが人間性を持ち、またその人間性から生まれるインタラクションを調整しなければならないことはいうまで'} +{'generated_text': '人とAIが協調するためには、AIが判断すべきことを人間が決める必要がある。人工知能の目的は、人間の知性、記憶、理解、'} +""" +``` + +When using PyTorch. + +``` python +from transformers import AutoTokenizer, AutoModelForCausalLM + + +tokenizer = AutoTokenizer.from_pretrained("abeja/gpt2-large-japanese") +model = AutoModelForCausalLM.from_pretrained("abeja/gpt2-large-japanese") + +input_text = "人とAIが協調するためには、" +input_ids = tokenizer.encode(input_text, return_tensors="pt") + +gen_tokens = model.generate( + input_ids, + max_length=100, + do_sample=True, + num_return_sequences=3, + top_p=0.95, + top_k=50, + pad_token_id=tokenizer.pad_token_id +) +for gen_text in tokenizer.batch_decode(gen_tokens, skip_special_tokens=True): + print(gen_text) +``` + +When using TensorFlow. + +```python +from transformers import AutoTokenizer, TFAutoModelForCausalLM + + +tokenizer = AutoTokenizer.from_pretrained("abeja/gpt2-large-japanese") +model = TFAutoModelForCausalLM.from_pretrained("abeja/gpt2-large-japanese", from_pt=True) + +input_text = "人とAIが協調するためには、" +input_ids = tokenizer.encode(input_text, return_tensors="tf") + +gen_tokens = model.generate( + input_ids, + max_length=100, + do_sample=True, + num_return_sequences=3, + top_p=0.95, + top_k=50, + pad_token_id=tokenizer.pad_token_id +) +for gen_text in tokenizer.batch_decode(gen_tokens, skip_special_tokens=True): + print(gen_text) +``` + +# Dataset +The model was trained on [Japanese CC-100](http://data.statmt.org/cc-100/ja.txt.xz), [Japanese Wikipedia](https://dumps.wikimedia.org/other/cirrussearch), and [Japanese OSCAR](https://huggingface.co/datasets/oscar). + +# Tokenization +The model uses a [sentencepiece](https://github.com/google/sentencepiece)-based tokenizer, the vocabulary was trained on the Japanese Wikipedia. + +# Licenese +[The MIT license](https://opensource.org/licenses/MIT) diff --git a/config.json b/config.json new file mode 100644 index 0000000..d3c10d3 --- /dev/null +++ b/config.json @@ -0,0 +1,40 @@ +{ + "_name_or_path": "checkpoint-huggingface/gpt2-large-japanese", + "activation_function": "gelu_new", + "architectures": [ + "GPT2LMHeadModel" + ], + "attn_pdrop": 0.1, + "bos_token_id": 1, + "embd_pdrop": 0.1, + "eos_token_id": 2, + "initializer_range": 0.02, + "layer_norm_epsilon": 1e-05, + "model_type": "gpt2", + "n_ctx": 1024, + "n_embd": 1280, + "n_head": 20, + "n_inner": null, + "n_layer": 36, + "n_positions": 1024, + "reorder_and_upcast_attn": false, + "resid_pdrop": 0.1, + "scale_attn_by_inverse_layer_idx": false, + "scale_attn_weights": true, + "summary_activation": null, + "summary_first_dropout": 0.1, + "summary_proj_to_labels": true, + "summary_type": "cls_index", + "summary_use_proj": true, + "task_specific_params": { + "text-generation": { + "do_sample": true, + "max_length": 50 + } + }, + "tokenizer_class": "T5Tokenizer", + "torch_dtype": "float32", + "transformers_version": "4.19.2", + "use_cache": true, + "vocab_size": 32000 +} diff --git a/pytorch_model.bin b/pytorch_model.bin new file mode 100644 index 0000000..3599b51 --- /dev/null +++ b/pytorch_model.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eb5575dce810c32790f2806f28d1813627a7d55d6df6e378ab15b313c0365481 +size 3040548553 diff --git a/special_tokens_map.json b/special_tokens_map.json new file mode 100644 index 0000000..6d5b058 --- /dev/null +++ b/special_tokens_map.json @@ -0,0 +1 @@ +{"bos_token": "", "eos_token": "", "unk_token": "", "sep_token": "[SEP]", "pad_token": "[PAD]", "cls_token": "[CLS]", "mask_token": "[MASK]"} \ No newline at end of file diff --git a/spiece.model b/spiece.model new file mode 100644 index 0000000..6b68ea0 --- /dev/null +++ b/spiece.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1ebbc2b0963521ffc640f69782e4e27c7209b1a6fff8f8d3083f599f8cfe2abb +size 783792 diff --git a/tf_model.h5 b/tf_model.h5 new file mode 100644 index 0000000..769afab --- /dev/null +++ b/tf_model.h5 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:350e954b93459963bc2c0ea333cfa8b1b2bd3e72cb74688561763c654764b26a +size 3003153768 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..456c3d2 --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1 @@ +{"eos_token": "", "unk_token": "", "pad_token": "[PAD]", "extra_ids": 0, "additional_special_tokens": [], "sp_model_kwargs": {}, "bos_token": "", "cls_token": "[CLS]", "sep_token": "[SEP]", "mask_token": "[MASK]", "do_lower_case": true, "tokenizer_class": "T5Tokenizer"} \ No newline at end of file