From 2ad796c1292a60875c35e3ebba5cb52149f0711d Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Mon, 28 Sep 2026 12:58:18 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: gvadhul/gpt2-2layer-1m Source: Original Platform --- .gitattributes | 35 ++ README.md | 48 +++ config.json | 35 ++ generation_config.json | 9 + model.safetensors | 3 + tokenizer.json | 714 +++++++++++++++++++++++++++++++++++++++++ tokenizer_config.json | 9 + 7 files changed, 853 insertions(+) create mode 100644 .gitattributes create mode 100644 README.md create mode 100644 config.json create mode 100644 generation_config.json create mode 100644 model.safetensors create mode 100644 tokenizer.json create mode 100644 tokenizer_config.json diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..a6344aa --- /dev/null +++ b/.gitattributes @@ -0,0 +1,35 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..3df8058 --- /dev/null +++ b/README.md @@ -0,0 +1,48 @@ +--- +language: +- en +license: apache-2.0 +tags: +- gpt2 +- causal-lm +- text-generation +- tiny +- testing +pipeline_tag: text-generation +--- + +# gpt2-2layer-1m + +A **randomly-initialized**, 2-layer GPT-2 model for functional / integration testing. +It produces incoherent text — that is by design. + +## Architecture + +| Hyperparameter | Value | +|---|---| +| Architecture | GPT-2 (decoder-only) | +| Layers | 2 | +| Hidden size | 256 | +| Attention heads | 4 | +| FFN inner size | 1 024 | +| Context length | 512 | +| Vocabulary size | 183 (byte-level BPE) | +| Total params | ~1.76 M | +| Non-embedding params | ~1.58 M | + +## Usage + +```python +from transformers import GPT2LMHeadModel, PreTrainedTokenizerFast + +tokenizer = PreTrainedTokenizerFast.from_pretrained("gvadhul/gpt2-2layer-1m") +model = GPT2LMHeadModel.from_pretrained("gvadhul/gpt2-2layer-1m") + +inputs = tokenizer("Hello world", return_tensors="pt") +out = model.generate(**inputs, max_new_tokens=20) +print(tokenizer.decode(out[0])) +``` + +## Intended use +Functional / unit testing of pipelines that need a *tiny* causal LM. +**Not suitable for any real NLP task** — weights are random. diff --git a/config.json b/config.json new file mode 100644 index 0000000..9c40866 --- /dev/null +++ b/config.json @@ -0,0 +1,35 @@ +{ + "activation_function": "gelu_new", + "add_cross_attention": false, + "architectures": [ + "GPT2LMHeadModel" + ], + "attn_pdrop": 0.0, + "bos_token_id": 0, + "dtype": "float32", + "embd_pdrop": 0.0, + "eos_token_id": 0, + "initializer_range": 0.02, + "layer_norm_epsilon": 1e-05, + "model_type": "gpt2", + "n_ctx": 512, + "n_embd": 256, + "n_head": 4, + "n_inner": 1024, + "n_layer": 2, + "n_positions": 2048, + "pad_token_id": null, + "reorder_and_upcast_attn": false, + "resid_pdrop": 0.0, + "scale_attn_by_inverse_layer_idx": false, + "scale_attn_weights": true, + "summary_activation": null, + "summary_first_dropout": 0.1, + "summary_proj_to_labels": true, + "summary_type": "cls_index", + "summary_use_proj": true, + "tie_word_embeddings": true, + "transformers_version": "5.12.1", + "use_cache": true, + "vocab_size": 183 +} diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..0b14c36 --- /dev/null +++ b/generation_config.json @@ -0,0 +1,9 @@ +{ + "_from_model_config": true, + "bos_token_id": 0, + "eos_token_id": 0, + "output_attentions": false, + "output_hidden_states": false, + "transformers_version": "5.12.1", + "use_cache": true +} diff --git a/model.safetensors b/model.safetensors new file mode 100644 index 0000000..2a1fc5b --- /dev/null +++ b/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f1a8c777831a0a1438148a4040f123764d69d636d8eec98927596949793684dc +size 8607400 diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..ba1c91f --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,714 @@ +{ + "version": "1.0", + "truncation": null, + "padding": null, + "added_tokens": [ + { + "id": 0, + "content": "<|endoftext|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1, + "content": "<|unk|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + } + ], + "normalizer": null, + "pre_tokenizer": { + "type": "ByteLevel", + "add_prefix_space": false, + "trim_offsets": true, + "use_regex": true + }, + "post_processor": null, + "decoder": { + "type": "ByteLevel", + "add_prefix_space": true, + "trim_offsets": true, + "use_regex": true + }, + "model": { + "type": "BPE", + "dropout": null, + "unk_token": "<|unk|>", + "continuing_subword_prefix": null, + "end_of_word_suffix": null, + "fuse_unk": false, + "byte_fallback": false, + "ignore_merges": false, + "vocab": { + "<|endoftext|>": 0, + "<|unk|>": 1, + "!": 2, + ".": 3, + "H": 4, + "I": 5, + "O": 6, + "T": 7, + "a": 8, + "b": 9, + "c": 10, + "d": 11, + "e": 12, + "f": 13, + "g": 14, + "h": 15, + "i": 16, + "j": 17, + "k": 18, + "l": 19, + "m": 20, + "n": 21, + "o": 22, + "p": 23, + "q": 24, + "r": 25, + "s": 26, + "t": 27, + "u": 28, + "v": 29, + "w": 30, + "x": 31, + "y": 32, + "z": 33, + "Ċ": 34, + "Ġ": 35, + "Ġt": 36, + "Ġa": 37, + "el": 38, + "er": 39, + "Th": 40, + "del": 41, + "es": 42, + "fo": 43, + "in": 44, + "is": 45, + "la": 46, + "mo": 47, + "Ġla": 48, + "Ġmo": 49, + "Ġtes": 50, + "Ġmodel": 51, + "Ġtest": 52, + "al": 53, + "as": 54, + "ed": 55, + "on": 56, + "ra": 57, + "wo": 58, + "Ġb": 59, + "Ġo": 60, + "Ġu": 61, + "Ġfo": 62, + "Ġis": 63, + "Ġth": 64, + "Ġtin": 65, + "The": 66, + "Ġtiny": 67, + "Hel": 68, + "In": 69, + "On": 70, + "ag": 71, + "ce": 72, + "ck": 73, + "do": 74, + "eg": 75, + "eed": 76, + "gu": 77, + "has": 78, + "id": 79, + "im": 80, + "ion": 81, + "ick": 82, + "ju": 83, + "ld": 84, + "lo": 85, + "mp": 86, + "mer": 87, + "mal": 88, + "nd": 89, + "ns": 90, + "need": 91, + "ngu": 92, + "ow": 93, + "pon": 94, + "qu": 95, + "rld": 96, + "rmer": 97, + "row": 98, + "sed": 99, + "smal": 100, + "teg": 101, + "tion": 102, + "ut": 103, + "ver": 104, + "val": 105, + "was": 106, + "yer": 107, + "zy": 108, + "ĠTh": 109, + "Ġwo": 110, + "Ġdo": 111, + "Ġhas": 112, + "Ġju": 113, + "Ġneed": 114, + "Ġqu": 115, + "Ġsmal": 116, + "Ġval": 117, + "Ġwas": 118, + "Ġtra": 119, + "Ġtwo": 120, + "Ġtim": 121, + "Ġand": 122, + "ere": 123, + "former": 124, + "ing": 125, + "Ġlangu": 126, + "Ġlayer": 127, + "Ġlazy": 128, + "Ġtests": 129, + "Ġtesting": 130, + "ration": 131, + "Ġbrow": 132, + "Ġbut": 133, + "Ġof": 134, + "Ġover": 135, + "Ġupon": 136, + "Ġused": 137, + "Ġfor": 138, + "Ġfox": 139, + "Ġthe": 140, + "Ġthere": 141, + "Hello": 142, + "Integ": 143, + "Once": 144, + "age": 145, + "mps": 146, + "nsformer": 147, + "ĠThis": 148, + "Ġworld": 149, + "Ġdog": 150, + "Ġjumps": 151, + "Ġquick": 152, + "Ġsmall": 153, + "Ġvalid": 154, + "Ġtransformer": 155, + "Ġtime": 156, + "Ġlanguage": 157, + "Ġlayers": 158, + "Ġbrown": 159, + "Integration": 160 + }, + "merges": [ + [ + "Ġ", + "t" + ], + [ + "Ġ", + "a" + ], + [ + "e", + "l" + ], + [ + "e", + "r" + ], + [ + "T", + "h" + ], + [ + "d", + "el" + ], + [ + "e", + "s" + ], + [ + "f", + "o" + ], + [ + "i", + "n" + ], + [ + "i", + "s" + ], + [ + "l", + "a" + ], + [ + "m", + "o" + ], + [ + "Ġ", + "la" + ], + [ + "Ġ", + "mo" + ], + [ + "Ġt", + "es" + ], + [ + "Ġmo", + "del" + ], + [ + "Ġtes", + "t" + ], + [ + "a", + "l" + ], + [ + "a", + "s" + ], + [ + "e", + "d" + ], + [ + "o", + "n" + ], + [ + "r", + "a" + ], + [ + "w", + "o" + ], + [ + "Ġ", + "b" + ], + [ + "Ġ", + "o" + ], + [ + "Ġ", + "u" + ], + [ + "Ġ", + "fo" + ], + [ + "Ġ", + "is" + ], + [ + "Ġt", + "h" + ], + [ + "Ġt", + "in" + ], + [ + "Th", + "e" + ], + [ + "Ġtin", + "y" + ], + [ + "H", + "el" + ], + [ + "I", + "n" + ], + [ + "O", + "n" + ], + [ + "a", + "g" + ], + [ + "c", + "e" + ], + [ + "c", + "k" + ], + [ + "d", + "o" + ], + [ + "e", + "g" + ], + [ + "e", + "ed" + ], + [ + "g", + "u" + ], + [ + "h", + "as" + ], + [ + "i", + "d" + ], + [ + "i", + "m" + ], + [ + "i", + "on" + ], + [ + "i", + "ck" + ], + [ + "j", + "u" + ], + [ + "l", + "d" + ], + [ + "l", + "o" + ], + [ + "m", + "p" + ], + [ + "m", + "er" + ], + [ + "m", + "al" + ], + [ + "n", + "d" + ], + [ + "n", + "s" + ], + [ + "n", + "eed" + ], + [ + "n", + "gu" + ], + [ + "o", + "w" + ], + [ + "p", + "on" + ], + [ + "q", + "u" + ], + [ + "r", + "ld" + ], + [ + "r", + "mer" + ], + [ + "r", + "ow" + ], + [ + "s", + "ed" + ], + [ + "s", + "mal" + ], + [ + "t", + "eg" + ], + [ + "t", + "ion" + ], + [ + "u", + "t" + ], + [ + "v", + "er" + ], + [ + "v", + "al" + ], + [ + "w", + "as" + ], + [ + "y", + "er" + ], + [ + "z", + "y" + ], + [ + "Ġ", + "Th" + ], + [ + "Ġ", + "wo" + ], + [ + "Ġ", + "do" + ], + [ + "Ġ", + "has" + ], + [ + "Ġ", + "ju" + ], + [ + "Ġ", + "need" + ], + [ + "Ġ", + "qu" + ], + [ + "Ġ", + "smal" + ], + [ + "Ġ", + "val" + ], + [ + "Ġ", + "was" + ], + [ + "Ġt", + "ra" + ], + [ + "Ġt", + "wo" + ], + [ + "Ġt", + "im" + ], + [ + "Ġa", + "nd" + ], + [ + "er", + "e" + ], + [ + "fo", + "rmer" + ], + [ + "in", + "g" + ], + [ + "Ġla", + "ngu" + ], + [ + "Ġla", + "yer" + ], + [ + "Ġla", + "zy" + ], + [ + "Ġtest", + "s" + ], + [ + "Ġtest", + "ing" + ], + [ + "ra", + "tion" + ], + [ + "Ġb", + "row" + ], + [ + "Ġb", + "ut" + ], + [ + "Ġo", + "f" + ], + [ + "Ġo", + "ver" + ], + [ + "Ġu", + "pon" + ], + [ + "Ġu", + "sed" + ], + [ + "Ġfo", + "r" + ], + [ + "Ġfo", + "x" + ], + [ + "Ġth", + "e" + ], + [ + "Ġth", + "ere" + ], + [ + "Hel", + "lo" + ], + [ + "In", + "teg" + ], + [ + "On", + "ce" + ], + [ + "ag", + "e" + ], + [ + "mp", + "s" + ], + [ + "ns", + "former" + ], + [ + "ĠTh", + "is" + ], + [ + "Ġwo", + "rld" + ], + [ + "Ġdo", + "g" + ], + [ + "Ġju", + "mps" + ], + [ + "Ġqu", + "ick" + ], + [ + "Ġsmal", + "l" + ], + [ + "Ġval", + "id" + ], + [ + "Ġtra", + "nsformer" + ], + [ + "Ġtim", + "e" + ], + [ + "Ġlangu", + "age" + ], + [ + "Ġlayer", + "s" + ], + [ + "Ġbrow", + "n" + ], + [ + "Integ", + "ration" + ] + ] + } +} \ No newline at end of file diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..0497300 --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,9 @@ +{ + "bos_token": "<|endoftext|>", + "eos_token": "<|endoftext|>", + "unk_token": "<|unk|>", + "pad_token": "<|endoftext|>", + "model_max_length": 512, + "tokenizer_class": "PreTrainedTokenizerFast", + "add_prefix_space": false +} \ No newline at end of file