Ksjsjjdj/Tiny-LLM-Z-V5
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

初始化项目，由ModelHub XC社区提供模型

Browse Source 
Model: Ksjsjjdj/Tiny-LLM-Z-V5
Source: Original Platform
This commit is contained in:
ModelHub XC
2026-05-30 10:57:18 +08:00
commit 955958c4c1
42 changed files with 46212 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
.gitattributes vendored Normal file
View File

@@ -0,0 +1,35 @@
*.7z filter=lfs diff=lfs merge=lfs -text
*.arrow filter=lfs diff=lfs merge=lfs -text
*.bin filter=lfs diff=lfs merge=lfs -text
*.bz2 filter=lfs diff=lfs merge=lfs -text
*.ckpt filter=lfs diff=lfs merge=lfs -text
*.ftz filter=lfs diff=lfs merge=lfs -text
*.gz filter=lfs diff=lfs merge=lfs -text
*.h5 filter=lfs diff=lfs merge=lfs -text
*.joblib filter=lfs diff=lfs merge=lfs -text
*.lfs.* filter=lfs diff=lfs merge=lfs -text
*.mlmodel filter=lfs diff=lfs merge=lfs -text
*.model filter=lfs diff=lfs merge=lfs -text
*.msgpack filter=lfs diff=lfs merge=lfs -text
*.npy filter=lfs diff=lfs merge=lfs -text
*.npz filter=lfs diff=lfs merge=lfs -text
*.onnx filter=lfs diff=lfs merge=lfs -text
*.ot filter=lfs diff=lfs merge=lfs -text
*.parquet filter=lfs diff=lfs merge=lfs -text
*.pb filter=lfs diff=lfs merge=lfs -text
*.pickle filter=lfs diff=lfs merge=lfs -text
*.pkl filter=lfs diff=lfs merge=lfs -text
*.pt filter=lfs diff=lfs merge=lfs -text
*.pth filter=lfs diff=lfs merge=lfs -text
*.rar filter=lfs diff=lfs merge=lfs -text
*.safetensors filter=lfs diff=lfs merge=lfs -text
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
*.tar.* filter=lfs diff=lfs merge=lfs -text
*.tar filter=lfs diff=lfs merge=lfs -text
*.tflite filter=lfs diff=lfs merge=lfs -text
*.tgz filter=lfs diff=lfs merge=lfs -text
*.wasm filter=lfs diff=lfs merge=lfs -text
*.xz filter=lfs diff=lfs merge=lfs -text
*.zip filter=lfs diff=lfs merge=lfs -text
*.zst filter=lfs diff=lfs merge=lfs -text
*tfevents* filter=lfs diff=lfs merge=lfs -text

3
README.md Normal file
View File

@@ -0,0 +1,3 @@
---
library_name: transformers
---

35
config.json Normal file
View File

@@ -0,0 +1,35 @@
{
"_name_or_path": "Tiny-LLM-Z-V5",
"architectures": [
"Qwen2ForCausalLM"
],
"attention_bias": false,
"attention_dropout": 0.0,
"bos_token_id": 1,
"dtype": "float32",
"eos_token_id": 2,
"head_dim": 64,
"hidden_act": "silu",
"hidden_size": 512,
"initializer_range": 0.02,
"intermediate_size": 2048,
"max_position_embeddings": 512,
"max_window_layers": 28,
"mlp_bias": false,
"model_type": "qwen2",
"num_attention_heads": 8,
"num_hidden_layers": 1,
"num_key_value_heads": 8,
"pad_token_id": 3,
"pretraining_tp": 1,
"rms_norm_eps": 1e-05,
"rope_scaling": null,
"rope_theta": 10000.0,
"sliding_window": null,
"tie_word_embeddings": true,
"torch_dtype": "float32",
"transformers_version": "4.48.3",
"use_cache": false,
"use_sliding_window": false,
"vocab_size": 256
}

35
final_model_complete/config.json Normal file
View File

@@ -0,0 +1,35 @@
{
"_name_or_path": "Tiny-LLM-Z-V5",
"architectures": [
"Qwen2ForCausalLM"
],
"attention_bias": false,
"attention_dropout": 0.0,
"bos_token_id": 1,
"dtype": "float32",
"eos_token_id": 2,
"head_dim": 64,
"hidden_act": "silu",
"hidden_size": 512,
"initializer_range": 0.02,
"intermediate_size": 2048,
"max_position_embeddings": 512,
"max_window_layers": 28,
"mlp_bias": false,
"model_type": "Qwen2",
"num_attention_heads": 8,
"num_hidden_layers": 1,
"num_key_value_heads": 8,
"pad_token_id": 3,
"pretraining_tp": 1,
"rms_norm_eps": 1e-05,
"rope_scaling": null,
"rope_theta": 10000.0,
"sliding_window": null,
"tie_word_embeddings": true,
"torch_dtype": "float32",
"transformers_version": "4.48.3",
"use_cache": false,
"use_sliding_window": false,
"vocab_size": 256
}

8
final_model_complete/generation_config.json Normal file
View File

@@ -0,0 +1,8 @@
{
"_from_model_config": true,
"bos_token_id": 1,
"eos_token_id": 2,
"pad_token_id": 3,
"transformers_version": "4.48.3",
"use_cache": false
}

3
final_model_complete/model.safetensors Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:9481b1d9d3ffa4f80126c095acffeec511c416c6278a5ed4b3a341fae7b16b8d
size 17315288

6
final_model_complete/special_tokens_map.json Normal file
View File

@@ -0,0 +1,6 @@
{
"bos_token": "<|endoftext|>",
"eos_token": "<|endoftext|>",
"pad_token": "<|endoftext|>",
"unk_token": "<|endoftext|>"
}

951
final_model_complete/tokenizer.json Normal file
View File

@@ -0,0 +1,951 @@
{
"version": "1.0",
"truncation": null,
"padding": null,
"added_tokens": [
{
"id": 0,
"content": "<|endoftext|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 1,
"content": "<|im_start|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 2,
"content": "<|im_end|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
}
],
"normalizer": null,
"pre_tokenizer": {
"type": "ByteLevel",
"add_prefix_space": false,
"trim_offsets": true,
"use_regex": true
},
"post_processor": {
"type": "ByteLevel",
"add_prefix_space": true,
"trim_offsets": false,
"use_regex": true
},
"decoder": {
"type": "ByteLevel",
"add_prefix_space": true,
"trim_offsets": true,
"use_regex": true
},
"model": {
"type": "BPE",
"dropout": null,
"unk_token": null,
"continuing_subword_prefix": null,
"end_of_word_suffix": null,
"fuse_unk": false,
"byte_fallback": false,
"ignore_merges": false,
"vocab": {
"<|endoftext|>": 0,
"<|im_start|>": 1,
"<|im_end|>": 2,
"!": 3,
"\"": 4,
"#": 5,
"$": 6,
"%": 7,
"&": 8,
"'": 9,
"(": 10,
")": 11,
"*": 12,
"+": 13,
",": 14,
"-": 15,
".": 16,
"/": 17,
"0": 18,
"1": 19,
"2": 20,
"3": 21,
"4": 22,
"5": 23,
"6": 24,
"7": 25,
"8": 26,
"9": 27,
":": 28,
";": 29,
"<": 30,
"=": 31,
">": 32,
"?": 33,
"@": 34,
"A": 35,
"B": 36,
"C": 37,
"D": 38,
"E": 39,
"F": 40,
"G": 41,
"H": 42,
"I": 43,
"J": 44,
"K": 45,
"L": 46,
"M": 47,
"N": 48,
"O": 49,
"P": 50,
"Q": 51,
"R": 52,
"S": 53,
"T": 54,
"U": 55,
"V": 56,
"W": 57,
"X": 58,
"Y": 59,
"Z": 60,
"[": 61,
"\\": 62,
"]": 63,
"^": 64,
"_": 65,
"`": 66,
"a": 67,
"b": 68,
"c": 69,
"d": 70,
"e": 71,
"f": 72,
"g": 73,
"h": 74,
"i": 75,
"j": 76,
"k": 77,
"l": 78,
"m": 79,
"n": 80,
"o": 81,
"p": 82,
"q": 83,
"r": 84,
"s": 85,
"t": 86,
"u": 87,
"v": 88,
"w": 89,
"x": 90,
"y": 91,
"z": 92,
"{": 93,
"|": 94,
"}": 95,
"~": 96,
"Ċ": 97,
"Ġ": 98,
"Ġt": 99,
"Ġa": 100,
"in": 101,
"on": 102,
"he": 103,
"ti": 104,
"re": 105,
"Ġthe": 106,
"er": 107,
"Ġs": 108,
"Ġo": 109,
"en": 110,
"al": 111,
"Ġc": 112,
"or": 113,
"at": 114,
"tion": 115,
"es": 116,
"Ġof": 117,
"is": 118,
"Ġp": 119,
"an": 120,
"ed": 121,
"Ġf": 122,
"ar": 123,
"it": 124,
"ro": 125,
"ing": 126,
"Ġan": 127,
"Ġin": 128,
"Ġd": 129,
"Ġw": 130,
"Ġm": 131,
"ic": 132,
"le": 133,
"Ġb": 134,
"Ġand": 135,
"ation": 136,
"as": 137,
"Ġe": 138,
"el": 139,
"Ġto": 140,
"et": 141,
"ent": 142,
"Ġre": 143,
"st": 144,
"ct": 145,
"Ġth": 146,
"om": 147,
"im": 148,
"ul": 149,
"ra": 150,
"Ġn": 151,
"ve": 152,
"ly": 153,
"od": 154,
"ur": 155,
"un": 156,
"si": 157,
"ce": 158,
"Ġis": 159,
"Ġl": 160,
"Ġfor": 161,
"us": 162,
"ith": 163,
"ĠT": 164,
"ol": 165,
"Ġg": 166,
"Ġst": 167,
"Ġpro": 168,
"ch": 169,
"ow": 170,
"tic": 171,
"Ġcon": 172,
"qu": 173,
"Ġh": 174,
"per": 175,
"Ġon": 176,
"ig": 177,
"am": 178,
"res": 179,
"Ġwith": 180,
"Ġthat": 181,
"ĠW": 182,
"ver": 183,
"um": 184,
"Ġ$": 185,
"il": 186,
"Ġex": 187,
"ut": 188,
"se": 189,
"ot": 190,
"ate": 191,
"ec": 192,
"id": 193,
"em": 194,
"ity": 195,
"Ġwe": 196,
"pl": 197,
"ĠWe": 198,
"if": 199,
"Ġ(": 200,
"ab": 201,
"ge": 202,
"ĠThe": 203,
"ri": 204,
"ter": 205,
"the": 206,
"Ġal": 207,
"Ġas": 208,
"Ġbe": 209,
"Ġare": 210,
"ect": 211,
"omp": 212,
"ction": 213,
"ations": 214,
"ure": 215,
"ĠI": 216,
"Ġmod": 217,
"Ġv": 218,
"ir": 219,
"os": 220,
"Ġsu": 221,
"Ġby": 222,
"ts": 223,
"Ġsp": 224,
"pro": 225,
"und": 226,
"iz": 227,
"ac": 228,
"ical": 229,
"for": 230,
"ran": 231,
"ant": 232,
"Ġcomp": 233,
"di": 234,
"ain": 235,
"Ġwh": 236,
"ener": 237,
"ren": 238,
"rom": 239,
"Ġmodel": 240,
"Ġus": 241,
"ĠA": 242,
"Ġthis": 243,
"ist": 244,
"tive": 245,
"Ġres": 246,
"sion": 247,
"ag": 248,
"Ġde": 249,
"du": 250,
"Ġsh": 251,
"ess": 252,
"Ġat": 253,
"ĠS": 254,
"oc": 255
},
"merges": [
[
"Ġ",
"t"
],
[
"Ġ",
"a"
],
[
"i",
"n"
],
[
"o",
"n"
],
[
"h",
"e"
],
[
"t",
"i"
],
[
"r",
"e"
],
[
"Ġt",
"he"
],
[
"e",
"r"
],
[
"Ġ",
"s"
],
[
"Ġ",
"o"
],
[
"e",
"n"
],
[
"a",
"l"
],
[
"Ġ",
"c"
],
[
"o",
"r"
],
[
"a",
"t"
],
[
"ti",
"on"
],
[
"e",
"s"
],
[
"Ġo",
"f"
],
[
"i",
"s"
],
[
"Ġ",
"p"
],
[
"a",
"n"
],
[
"e",
"d"
],
[
"Ġ",
"f"
],
[
"a",
"r"
],
[
"i",
"t"
],
[
"r",
"o"
],
[
"in",
"g"
],
[
"Ġa",
"n"
],
[
"Ġ",
"in"
],
[
"Ġ",
"d"
],
[
"Ġ",
"w"
],
[
"Ġ",
"m"
],
[
"i",
"c"
],
[
"l",
"e"
],
[
"Ġ",
"b"
],
[
"Ġan",
"d"
],
[
"a",
"tion"
],
[
"a",
"s"
],
[
"Ġ",
"e"
],
[
"e",
"l"
],
[
"Ġt",
"o"
],
[
"e",
"t"
],
[
"en",
"t"
],
[
"Ġ",
"re"
],
[
"s",
"t"
],
[
"c",
"t"
],
[
"Ġt",
"h"
],
[
"o",
"m"
],
[
"i",
"m"
],
[
"u",
"l"
],
[
"r",
"a"
],
[
"Ġ",
"n"
],
[
"v",
"e"
],
[
"l",
"y"
],
[
"o",
"d"
],
[
"u",
"r"
],
[
"u",
"n"
],
[
"s",
"i"
],
[
"c",
"e"
],
[
"Ġ",
"is"
],
[
"Ġ",
"l"
],
[
"Ġf",
"or"
],
[
"u",
"s"
],
[
"it",
"h"
],
[
"Ġ",
"T"
],
[
"o",
"l"
],
[
"Ġ",
"g"
],
[
"Ġs",
"t"
],
[
"Ġp",
"ro"
],
[
"c",
"h"
],
[
"o",
"w"
],
[
"ti",
"c"
],
[
"Ġc",
"on"
],
[
"q",
"u"
],
[
"Ġ",
"h"
],
[
"p",
"er"
],
[
"Ġ",
"on"
],
[
"i",
"g"
],
[
"a",
"m"
],
[
"re",
"s"
],
[
"Ġw",
"ith"
],
[
"Ġth",
"at"
],
[
"Ġ",
"W"
],
[
"v",
"er"
],
[
"u",
"m"
],
[
"Ġ",
"$"
],
[
"i",
"l"
],
[
"Ġe",
"x"
],
[
"u",
"t"
],
[
"s",
"e"
],
[
"o",
"t"
],
[
"at",
"e"
],
[
"e",
"c"
],
[
"i",
"d"
],
[
"e",
"m"
],
[
"it",
"y"
],
[
"Ġw",
"e"
],
[
"p",
"l"
],
[
"ĠW",
"e"
],
[
"i",
"f"
],
[
"Ġ",
"("
],
[
"a",
"b"
],
[
"g",
"e"
],
[
"ĠT",
"he"
],
[
"r",
"i"
],
[
"t",
"er"
],
[
"t",
"he"
],
[
"Ġa",
"l"
],
[
"Ġa",
"s"
],
[
"Ġb",
"e"
],
[
"Ġa",
"re"
],
[
"e",
"ct"
],
[
"om",
"p"
],
[
"c",
"tion"
],
[
"ation",
"s"
],
[
"u",
"re"
],
[
"Ġ",
"I"
],
[
"Ġm",
"od"
],
[
"Ġ",
"v"
],
[
"i",
"r"
],
[
"o",
"s"
],
[
"Ġs",
"u"
],
[
"Ġb",
"y"
],
[
"t",
"s"
],
[
"Ġs",
"p"
],
[
"p",
"ro"
],
[
"un",
"d"
],
[
"i",
"z"
],
[
"a",
"c"
],
[
"ic",
"al"
],
[
"f",
"or"
],
[
"r",
"an"
],
[
"an",
"t"
],
[
"Ġc",
"omp"
],
[
"d",
"i"
],
[
"a",
"in"
],
[
"Ġw",
"h"
],
[
"en",
"er"
],
[
"re",
"n"
],
[
"ro",
"m"
],
[
"Ġmod",
"el"
],
[
"Ġ",
"us"
],
[
"Ġ",
"A"
],
[
"Ġth",
"is"
],
[
"is",
"t"
],
[
"ti",
"ve"
],
[
"Ġre",
"s"
],
[
"si",
"on"
],
[
"a",
"g"
],
[
"Ġd",
"e"
],
[
"d",
"u"
],
[
"Ġs",
"h"
],
[
"es",
"s"
],
[
"Ġa",
"t"
],
[
"Ġ",
"S"
],
[
"o",
"c"
]
]
}
}

41
final_model_complete/tokenizer_config.json Normal file
View File

@@ -0,0 +1,41 @@
{
"added_tokens_decoder": {
"0": {
"content": "<|endoftext|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"1": {
"content": "<|im_start|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"2": {
"content": "<|im_end|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
}
},
"bos_token": "<|endoftext|>",
"chat_template": "{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
"clean_up_tokenization_spaces": false,
"eos_token": "<|endoftext|>",
"extra_special_tokens": {},
"model_input_names": [
"input_ids",
"attention_mask"
],
"model_max_length": 1000000000000000019884624838656,
"pad_token": "<|endoftext|>",
"tokenizer_class": "PreTrainedTokenizerFast",
"unk_token": "<|endoftext|>"
}

3
final_model_complete/training_args.bin Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:3094fcb2cbf0c2dbb4dd559e2ad79c9e24d99aeecd652bbb12aa828bc03d3da2
size 5713

8
generation_config.json Normal file
View File

@@ -0,0 +1,8 @@
{
"_from_model_config": true,
"bos_token_id": 1,
"eos_token_id": 2,
"pad_token_id": 3,
"transformers_version": "4.48.3",
"use_cache": false
}

3
model.safetensors Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:9481b1d9d3ffa4f80126c095acffeec511c416c6278a5ed4b3a341fae7b16b8d
size 17315288

3
optimizer.pt Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:faad0c8b87757094274de331ad8ff25edfdae71e0ad7d87baebc418843db2fed
size 34640005

3
rng_state.pth Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:c1971a7f5a4b2dbf37c7eaedcc9ddd90b7f2853c5557f35719c1a2082903b821
size 14455

3
scheduler.pt Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:8a9f154b1359afdda4d3ed65870a0e8dc4d46259d77c6ba1858bc2095aee8d67
size 1465

6
special_tokens_map.json Normal file
View File

@@ -0,0 +1,6 @@
{
"bos_token": "<|endoftext|>",
"eos_token": "<|endoftext|>",
"pad_token": "<|endoftext|>",
"unk_token": "<|endoftext|>"
}

35
tiny_llm_pretrain/checkpoint-10000/config.json Normal file
View File

@@ -0,0 +1,35 @@
{
"_name_or_path": "Tiny-LLM-Z-V5",
"architectures": [
"Qwen2ForCausalLM"
],
"attention_bias": false,
"attention_dropout": 0.0,
"bos_token_id": 1,
"dtype": "float32",
"eos_token_id": 2,
"head_dim": 64,
"hidden_act": "silu",
"hidden_size": 512,
"initializer_range": 0.02,
"intermediate_size": 2048,
"max_position_embeddings": 512,
"max_window_layers": 28,
"mlp_bias": false,
"model_type": "qwen2",
"num_attention_heads": 8,
"num_hidden_layers": 1,
"num_key_value_heads": 8,
"pad_token_id": 3,
"pretraining_tp": 1,
"rms_norm_eps": 1e-05,
"rope_scaling": null,
"rope_theta": 10000.0,
"sliding_window": null,
"tie_word_embeddings": true,
"torch_dtype": "float32",
"transformers_version": "4.48.3",
"use_cache": false,
"use_sliding_window": false,
"vocab_size": 256
}

8
tiny_llm_pretrain/checkpoint-10000/generation_config.json Normal file
View File

@@ -0,0 +1,8 @@
{
"_from_model_config": true,
"bos_token_id": 1,
"eos_token_id": 2,
"pad_token_id": 3,
"transformers_version": "4.48.3",
"use_cache": false
}

3
tiny_llm_pretrain/checkpoint-10000/model.safetensors Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:9481b1d9d3ffa4f80126c095acffeec511c416c6278a5ed4b3a341fae7b16b8d
size 17315288

3
tiny_llm_pretrain/checkpoint-10000/optimizer.pt Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:faad0c8b87757094274de331ad8ff25edfdae71e0ad7d87baebc418843db2fed
size 34640005

3
tiny_llm_pretrain/checkpoint-10000/rng_state.pth Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:c1971a7f5a4b2dbf37c7eaedcc9ddd90b7f2853c5557f35719c1a2082903b821
size 14455

3
tiny_llm_pretrain/checkpoint-10000/scheduler.pt Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:8a9f154b1359afdda4d3ed65870a0e8dc4d46259d77c6ba1858bc2095aee8d67
size 1465

6
tiny_llm_pretrain/checkpoint-10000/special_tokens_map.json Normal file
View File

@@ -0,0 +1,6 @@
{
"bos_token": "<|endoftext|>",
"eos_token": "<|endoftext|>",
"pad_token": "<|endoftext|>",
"unk_token": "<|endoftext|>"
}

951
tiny_llm_pretrain/checkpoint-10000/tokenizer.json Normal file
View File

@@ -0,0 +1,951 @@
{
"version": "1.0",
"truncation": null,
"padding": null,
"added_tokens": [
{
"id": 0,
"content": "<|endoftext|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 1,
"content": "<|im_start|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 2,
"content": "<|im_end|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
}
],
"normalizer": null,
"pre_tokenizer": {
"type": "ByteLevel",
"add_prefix_space": false,
"trim_offsets": true,
"use_regex": true
},
"post_processor": {
"type": "ByteLevel",
"add_prefix_space": true,
"trim_offsets": false,
"use_regex": true
},
"decoder": {
"type": "ByteLevel",
"add_prefix_space": true,
"trim_offsets": true,
"use_regex": true
},
"model": {
"type": "BPE",
"dropout": null,
"unk_token": null,
"continuing_subword_prefix": null,
"end_of_word_suffix": null,
"fuse_unk": false,
"byte_fallback": false,
"ignore_merges": false,
"vocab": {
"<|endoftext|>": 0,
"<|im_start|>": 1,
"<|im_end|>": 2,
"!": 3,
"\"": 4,
"#": 5,
"$": 6,
"%": 7,
"&": 8,
"'": 9,
"(": 10,
")": 11,
"*": 12,
"+": 13,
",": 14,
"-": 15,
".": 16,
"/": 17,
"0": 18,
"1": 19,
"2": 20,
"3": 21,
"4": 22,
"5": 23,
"6": 24,
"7": 25,
"8": 26,
"9": 27,
":": 28,
";": 29,
"<": 30,
"=": 31,
">": 32,
"?": 33,
"@": 34,
"A": 35,
"B": 36,
"C": 37,
"D": 38,
"E": 39,
"F": 40,
"G": 41,
"H": 42,
"I": 43,
"J": 44,
"K": 45,
"L": 46,
"M": 47,
"N": 48,
"O": 49,
"P": 50,
"Q": 51,
"R": 52,
"S": 53,
"T": 54,
"U": 55,
"V": 56,
"W": 57,
"X": 58,
"Y": 59,
"Z": 60,
"[": 61,
"\\": 62,
"]": 63,
"^": 64,
"_": 65,
"`": 66,
"a": 67,
"b": 68,
"c": 69,
"d": 70,
"e": 71,
"f": 72,
"g": 73,
"h": 74,
"i": 75,
"j": 76,
"k": 77,
"l": 78,
"m": 79,
"n": 80,
"o": 81,
"p": 82,
"q": 83,
"r": 84,
"s": 85,
"t": 86,
"u": 87,
"v": 88,
"w": 89,
"x": 90,
"y": 91,
"z": 92,
"{": 93,
"|": 94,
"}": 95,
"~": 96,
"Ċ": 97,
"Ġ": 98,
"Ġt": 99,
"Ġa": 100,
"in": 101,
"on": 102,
"he": 103,
"ti": 104,
"re": 105,
"Ġthe": 106,
"er": 107,
"Ġs": 108,
"Ġo": 109,
"en": 110,
"al": 111,
"Ġc": 112,
"or": 113,
"at": 114,
"tion": 115,
"es": 116,
"Ġof": 117,
"is": 118,
"Ġp": 119,
"an": 120,
"ed": 121,
"Ġf": 122,
"ar": 123,
"it": 124,
"ro": 125,
"ing": 126,
"Ġan": 127,
"Ġin": 128,
"Ġd": 129,
"Ġw": 130,
"Ġm": 131,
"ic": 132,
"le": 133,
"Ġb": 134,
"Ġand": 135,
"ation": 136,
"as": 137,
"Ġe": 138,
"el": 139,
"Ġto": 140,
"et": 141,
"ent": 142,
"Ġre": 143,
"st": 144,
"ct": 145,
"Ġth": 146,
"om": 147,
"im": 148,
"ul": 149,
"ra": 150,
"Ġn": 151,
"ve": 152,
"ly": 153,
"od": 154,
"ur": 155,
"un": 156,
"si": 157,
"ce": 158,
"Ġis": 159,
"Ġl": 160,
"Ġfor": 161,
"us": 162,
"ith": 163,
"ĠT": 164,
"ol": 165,
"Ġg": 166,
"Ġst": 167,
"Ġpro": 168,
"ch": 169,
"ow": 170,
"tic": 171,
"Ġcon": 172,
"qu": 173,
"Ġh": 174,
"per": 175,
"Ġon": 176,
"ig": 177,
"am": 178,
"res": 179,
"Ġwith": 180,
"Ġthat": 181,
"ĠW": 182,
"ver": 183,
"um": 184,
"Ġ$": 185,
"il": 186,
"Ġex": 187,
"ut": 188,
"se": 189,
"ot": 190,
"ate": 191,
"ec": 192,
"id": 193,
"em": 194,
"ity": 195,
"Ġwe": 196,
"pl": 197,
"ĠWe": 198,
"if": 199,
"Ġ(": 200,
"ab": 201,
"ge": 202,
"ĠThe": 203,
"ri": 204,
"ter": 205,
"the": 206,
"Ġal": 207,
"Ġas": 208,
"Ġbe": 209,
"Ġare": 210,
"ect": 211,
"omp": 212,
"ction": 213,
"ations": 214,
"ure": 215,
"ĠI": 216,
"Ġmod": 217,
"Ġv": 218,
"ir": 219,
"os": 220,
"Ġsu": 221,
"Ġby": 222,
"ts": 223,
"Ġsp": 224,
"pro": 225,
"und": 226,
"iz": 227,
"ac": 228,
"ical": 229,
"for": 230,
"ran": 231,
"ant": 232,
"Ġcomp": 233,
"di": 234,
"ain": 235,
"Ġwh": 236,
"ener": 237,
"ren": 238,
"rom": 239,
"Ġmodel": 240,
"Ġus": 241,
"ĠA": 242,
"Ġthis": 243,
"ist": 244,
"tive": 245,
"Ġres": 246,
"sion": 247,
"ag": 248,
"Ġde": 249,
"du": 250,
"Ġsh": 251,
"ess": 252,
"Ġat": 253,
"ĠS": 254,
"oc": 255
},
"merges": [
[
"Ġ",
"t"
],
[
"Ġ",
"a"
],
[
"i",
"n"
],
[
"o",
"n"
],
[
"h",
"e"
],
[
"t",
"i"
],
[
"r",
"e"
],
[
"Ġt",
"he"
],
[
"e",
"r"
],
[
"Ġ",
"s"
],
[
"Ġ",
"o"
],
[
"e",
"n"
],
[
"a",
"l"
],
[
"Ġ",
"c"
],
[
"o",
"r"
],
[
"a",
"t"
],
[
"ti",
"on"
],
[
"e",
"s"
],
[
"Ġo",
"f"
],
[
"i",
"s"
],
[
"Ġ",
"p"
],
[
"a",
"n"
],
[
"e",
"d"
],
[
"Ġ",
"f"
],
[
"a",
"r"
],
[
"i",
"t"
],
[
"r",
"o"
],
[
"in",
"g"
],
[
"Ġa",
"n"
],
[
"Ġ",
"in"
],
[
"Ġ",
"d"
],
[
"Ġ",
"w"
],
[
"Ġ",
"m"
],
[
"i",
"c"
],
[
"l",
"e"
],
[
"Ġ",
"b"
],
[
"Ġan",
"d"
],
[
"a",
"tion"
],
[
"a",
"s"
],
[
"Ġ",
"e"
],
[
"e",
"l"
],
[
"Ġt",
"o"
],
[
"e",
"t"
],
[
"en",
"t"
],
[
"Ġ",
"re"
],
[
"s",
"t"
],
[
"c",
"t"
],
[
"Ġt",
"h"
],
[
"o",
"m"
],
[
"i",
"m"
],
[
"u",
"l"
],
[
"r",
"a"
],
[
"Ġ",
"n"
],
[
"v",
"e"
],
[
"l",
"y"
],
[
"o",
"d"
],
[
"u",
"r"
],
[
"u",
"n"
],
[
"s",
"i"
],
[
"c",
"e"
],
[
"Ġ",
"is"
],
[
"Ġ",
"l"
],
[
"Ġf",
"or"
],
[
"u",
"s"
],
[
"it",
"h"
],
[
"Ġ",
"T"
],
[
"o",
"l"
],
[
"Ġ",
"g"
],
[
"Ġs",
"t"
],
[
"Ġp",
"ro"
],
[
"c",
"h"
],
[
"o",
"w"
],
[
"ti",
"c"
],
[
"Ġc",
"on"
],
[
"q",
"u"
],
[
"Ġ",
"h"
],
[
"p",
"er"
],
[
"Ġ",
"on"
],
[
"i",
"g"
],
[
"a",
"m"
],
[
"re",
"s"
],
[
"Ġw",
"ith"
],
[
"Ġth",
"at"
],
[
"Ġ",
"W"
],
[
"v",
"er"
],
[
"u",
"m"
],
[
"Ġ",
"$"
],
[
"i",
"l"
],
[
"Ġe",
"x"
],
[
"u",
"t"
],
[
"s",
"e"
],
[
"o",
"t"
],
[
"at",
"e"
],
[
"e",
"c"
],
[
"i",
"d"
],
[
"e",
"m"
],
[
"it",
"y"
],
[
"Ġw",
"e"
],
[
"p",
"l"
],
[
"ĠW",
"e"
],
[
"i",
"f"
],
[
"Ġ",
"("
],
[
"a",
"b"
],
[
"g",
"e"
],
[
"ĠT",
"he"
],
[
"r",
"i"
],
[
"t",
"er"
],
[
"t",
"he"
],
[
"Ġa",
"l"
],
[
"Ġa",
"s"
],
[
"Ġb",
"e"
],
[
"Ġa",
"re"
],
[
"e",
"ct"
],
[
"om",
"p"
],
[
"c",
"tion"
],
[
"ation",
"s"
],
[
"u",
"re"
],
[
"Ġ",
"I"
],
[
"Ġm",
"od"
],
[
"Ġ",
"v"
],
[
"i",
"r"
],
[
"o",
"s"
],
[
"Ġs",
"u"
],
[
"Ġb",
"y"
],
[
"t",
"s"
],
[
"Ġs",
"p"
],
[
"p",
"ro"
],
[
"un",
"d"
],
[
"i",
"z"
],
[
"a",
"c"
],
[
"ic",
"al"
],
[
"f",
"or"
],
[
"r",
"an"
],
[
"an",
"t"
],
[
"Ġc",
"omp"
],
[
"d",
"i"
],
[
"a",
"in"
],
[
"Ġw",
"h"
],
[
"en",
"er"
],
[
"re",
"n"
],
[
"ro",
"m"
],
[
"Ġmod",
"el"
],
[
"Ġ",
"us"
],
[
"Ġ",
"A"
],
[
"Ġth",
"is"
],
[
"is",
"t"
],
[
"ti",
"ve"
],
[
"Ġre",
"s"
],
[
"si",
"on"
],
[
"a",
"g"
],
[
"Ġd",
"e"
],
[
"d",
"u"
],
[
"Ġs",
"h"
],
[
"es",
"s"
],
[
"Ġa",
"t"
],
[
"Ġ",
"S"
],
[
"o",
"c"
]
]
}
}

41
tiny_llm_pretrain/checkpoint-10000/tokenizer_config.json Normal file
View File

@@ -0,0 +1,41 @@
{
"added_tokens_decoder": {
"0": {
"content": "<|endoftext|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"1": {
"content": "<|im_start|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"2": {
"content": "<|im_end|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
}
},
"bos_token": "<|endoftext|>",
"chat_template": "{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
"clean_up_tokenization_spaces": false,
"eos_token": "<|endoftext|>",
"extra_special_tokens": {},
"model_input_names": [
"input_ids",
"attention_mask"
],
"model_max_length": 1000000000000000019884624838656,
"pad_token": "<|endoftext|>",
"tokenizer_class": "PreTrainedTokenizerFast",
"unk_token": "<|endoftext|>"
}

14033
tiny_llm_pretrain/checkpoint-10000/trainer_state.json Normal file
View File

File diff suppressed because it is too large Load Diff

3
tiny_llm_pretrain/checkpoint-10000/training_args.bin Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:3094fcb2cbf0c2dbb4dd559e2ad79c9e24d99aeecd652bbb12aa828bc03d3da2
size 5713

35
tiny_llm_pretrain/checkpoint-9900/config.json Normal file
View File

@@ -0,0 +1,35 @@
{
"_name_or_path": "Tiny-LLM-Z-V5",
"architectures": [
"Qwen2ForCausalLM"
],
"attention_bias": false,
"attention_dropout": 0.0,
"bos_token_id": 1,
"dtype": "float32",
"eos_token_id": 2,
"head_dim": 64,
"hidden_act": "silu",
"hidden_size": 512,
"initializer_range": 0.02,
"intermediate_size": 2048,
"max_position_embeddings": 512,
"max_window_layers": 28,
"mlp_bias": false,
"model_type": "qwen2",
"num_attention_heads": 8,
"num_hidden_layers": 1,
"num_key_value_heads": 8,
"pad_token_id": 3,
"pretraining_tp": 1,
"rms_norm_eps": 1e-05,
"rope_scaling": null,
"rope_theta": 10000.0,
"sliding_window": null,
"tie_word_embeddings": true,
"torch_dtype": "float32",
"transformers_version": "4.48.3",
"use_cache": false,
"use_sliding_window": false,
"vocab_size": 256
}

8
tiny_llm_pretrain/checkpoint-9900/generation_config.json Normal file
View File

@@ -0,0 +1,8 @@
{
"_from_model_config": true,
"bos_token_id": 1,
"eos_token_id": 2,
"pad_token_id": 3,
"transformers_version": "4.48.3",
"use_cache": false
}

3
tiny_llm_pretrain/checkpoint-9900/model.safetensors Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:b27284ac93718ba2efbfb92e1e76368cdf3eae27f6c1db5a6e83dc24c93b48a4
size 17315288

3
tiny_llm_pretrain/checkpoint-9900/optimizer.pt Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:bd3e7422bdaed1c99ace97a59ee33c41f5f43f6807d6c74a7a01968d6b862705
size 34640005

3
tiny_llm_pretrain/checkpoint-9900/rng_state.pth Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:c1971a7f5a4b2dbf37c7eaedcc9ddd90b7f2853c5557f35719c1a2082903b821
size 14455

3
tiny_llm_pretrain/checkpoint-9900/scheduler.pt Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:fa4c7a3f0413da0222033791fbd61f87fa01f48b28114defa805cbadbcde3d7c
size 1465

6
tiny_llm_pretrain/checkpoint-9900/special_tokens_map.json Normal file
View File

@@ -0,0 +1,6 @@
{
"bos_token": "<|endoftext|>",
"eos_token": "<|endoftext|>",
"pad_token": "<|endoftext|>",
"unk_token": "<|endoftext|>"
}

951
tiny_llm_pretrain/checkpoint-9900/tokenizer.json Normal file
View File

@@ -0,0 +1,951 @@
{
"version": "1.0",
"truncation": null,
"padding": null,
"added_tokens": [
{
"id": 0,
"content": "<|endoftext|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 1,
"content": "<|im_start|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 2,
"content": "<|im_end|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
}
],
"normalizer": null,
"pre_tokenizer": {
"type": "ByteLevel",
"add_prefix_space": false,
"trim_offsets": true,
"use_regex": true
},
"post_processor": {
"type": "ByteLevel",
"add_prefix_space": true,
"trim_offsets": false,
"use_regex": true
},
"decoder": {
"type": "ByteLevel",
"add_prefix_space": true,
"trim_offsets": true,
"use_regex": true
},
"model": {
"type": "BPE",
"dropout": null,
"unk_token": null,
"continuing_subword_prefix": null,
"end_of_word_suffix": null,
"fuse_unk": false,
"byte_fallback": false,
"ignore_merges": false,
"vocab": {
"<|endoftext|>": 0,
"<|im_start|>": 1,
"<|im_end|>": 2,
"!": 3,
"\"": 4,
"#": 5,
"$": 6,
"%": 7,
"&": 8,
"'": 9,
"(": 10,
")": 11,
"*": 12,
"+": 13,
",": 14,
"-": 15,
".": 16,
"/": 17,
"0": 18,
"1": 19,
"2": 20,
"3": 21,
"4": 22,
"5": 23,
"6": 24,
"7": 25,
"8": 26,
"9": 27,
":": 28,
";": 29,
"<": 30,
"=": 31,
">": 32,
"?": 33,
"@": 34,
"A": 35,
"B": 36,
"C": 37,
"D": 38,
"E": 39,
"F": 40,
"G": 41,
"H": 42,
"I": 43,
"J": 44,
"K": 45,
"L": 46,
"M": 47,
"N": 48,
"O": 49,
"P": 50,
"Q": 51,
"R": 52,
"S": 53,
"T": 54,
"U": 55,
"V": 56,
"W": 57,
"X": 58,
"Y": 59,
"Z": 60,
"[": 61,
"\\": 62,
"]": 63,
"^": 64,
"_": 65,
"`": 66,
"a": 67,
"b": 68,
"c": 69,
"d": 70,
"e": 71,
"f": 72,
"g": 73,
"h": 74,
"i": 75,
"j": 76,
"k": 77,
"l": 78,
"m": 79,
"n": 80,
"o": 81,
"p": 82,
"q": 83,
"r": 84,
"s": 85,
"t": 86,
"u": 87,
"v": 88,
"w": 89,
"x": 90,
"y": 91,
"z": 92,
"{": 93,
"|": 94,
"}": 95,
"~": 96,
"Ċ": 97,
"Ġ": 98,
"Ġt": 99,
"Ġa": 100,
"in": 101,
"on": 102,
"he": 103,
"ti": 104,
"re": 105,
"Ġthe": 106,
"er": 107,
"Ġs": 108,
"Ġo": 109,
"en": 110,
"al": 111,
"Ġc": 112,
"or": 113,
"at": 114,
"tion": 115,
"es": 116,
"Ġof": 117,
"is": 118,
"Ġp": 119,
"an": 120,
"ed": 121,
"Ġf": 122,
"ar": 123,
"it": 124,
"ro": 125,
"ing": 126,
"Ġan": 127,
"Ġin": 128,
"Ġd": 129,
"Ġw": 130,
"Ġm": 131,
"ic": 132,
"le": 133,
"Ġb": 134,
"Ġand": 135,
"ation": 136,
"as": 137,
"Ġe": 138,
"el": 139,
"Ġto": 140,
"et": 141,
"ent": 142,
"Ġre": 143,
"st": 144,
"ct": 145,
"Ġth": 146,
"om": 147,
"im": 148,
"ul": 149,
"ra": 150,
"Ġn": 151,
"ve": 152,
"ly": 153,
"od": 154,
"ur": 155,
"un": 156,
"si": 157,
"ce": 158,
"Ġis": 159,
"Ġl": 160,
"Ġfor": 161,
"us": 162,
"ith": 163,
"ĠT": 164,
"ol": 165,
"Ġg": 166,
"Ġst": 167,
"Ġpro": 168,
"ch": 169,
"ow": 170,
"tic": 171,
"Ġcon": 172,
"qu": 173,
"Ġh": 174,
"per": 175,
"Ġon": 176,
"ig": 177,
"am": 178,
"res": 179,
"Ġwith": 180,
"Ġthat": 181,
"ĠW": 182,
"ver": 183,
"um": 184,
"Ġ$": 185,
"il": 186,
"Ġex": 187,
"ut": 188,
"se": 189,
"ot": 190,
"ate": 191,
"ec": 192,
"id": 193,
"em": 194,
"ity": 195,
"Ġwe": 196,
"pl": 197,
"ĠWe": 198,
"if": 199,
"Ġ(": 200,
"ab": 201,
"ge": 202,
"ĠThe": 203,
"ri": 204,
"ter": 205,
"the": 206,
"Ġal": 207,
"Ġas": 208,
"Ġbe": 209,
"Ġare": 210,
"ect": 211,
"omp": 212,
"ction": 213,
"ations": 214,
"ure": 215,
"ĠI": 216,
"Ġmod": 217,
"Ġv": 218,
"ir": 219,
"os": 220,
"Ġsu": 221,
"Ġby": 222,
"ts": 223,
"Ġsp": 224,
"pro": 225,
"und": 226,
"iz": 227,
"ac": 228,
"ical": 229,
"for": 230,
"ran": 231,
"ant": 232,
"Ġcomp": 233,
"di": 234,
"ain": 235,
"Ġwh": 236,
"ener": 237,
"ren": 238,
"rom": 239,
"Ġmodel": 240,
"Ġus": 241,
"ĠA": 242,
"Ġthis": 243,
"ist": 244,
"tive": 245,
"Ġres": 246,
"sion": 247,
"ag": 248,
"Ġde": 249,
"du": 250,
"Ġsh": 251,
"ess": 252,
"Ġat": 253,
"ĠS": 254,
"oc": 255
},
"merges": [
[
"Ġ",
"t"
],
[
"Ġ",
"a"
],
[
"i",
"n"
],
[
"o",
"n"
],
[
"h",
"e"
],
[
"t",
"i"
],
[
"r",
"e"
],
[
"Ġt",
"he"
],
[
"e",
"r"
],
[
"Ġ",
"s"
],
[
"Ġ",
"o"
],
[
"e",
"n"
],
[
"a",
"l"
],
[
"Ġ",
"c"
],
[
"o",
"r"
],
[
"a",
"t"
],
[
"ti",
"on"
],
[
"e",
"s"
],
[
"Ġo",
"f"
],
[
"i",
"s"
],
[
"Ġ",
"p"
],
[
"a",
"n"
],
[
"e",
"d"
],
[
"Ġ",
"f"
],
[
"a",
"r"
],
[
"i",
"t"
],
[
"r",
"o"
],
[
"in",
"g"
],
[
"Ġa",
"n"
],
[
"Ġ",
"in"
],
[
"Ġ",
"d"
],
[
"Ġ",
"w"
],
[
"Ġ",
"m"
],
[
"i",
"c"
],
[
"l",
"e"
],
[
"Ġ",
"b"
],
[
"Ġan",
"d"
],
[
"a",
"tion"
],
[
"a",
"s"
],
[
"Ġ",
"e"
],
[
"e",
"l"
],
[
"Ġt",
"o"
],
[
"e",
"t"
],
[
"en",
"t"
],
[
"Ġ",
"re"
],
[
"s",
"t"
],
[
"c",
"t"
],
[
"Ġt",
"h"
],
[
"o",
"m"
],
[
"i",
"m"
],
[
"u",
"l"
],
[
"r",
"a"
],
[
"Ġ",
"n"
],
[
"v",
"e"
],
[
"l",
"y"
],
[
"o",
"d"
],
[
"u",
"r"
],
[
"u",
"n"
],
[
"s",
"i"
],
[
"c",
"e"
],
[
"Ġ",
"is"
],
[
"Ġ",
"l"
],
[
"Ġf",
"or"
],
[
"u",
"s"
],
[
"it",
"h"
],
[
"Ġ",
"T"
],
[
"o",
"l"
],
[
"Ġ",
"g"
],
[
"Ġs",
"t"
],
[
"Ġp",
"ro"
],
[
"c",
"h"
],
[
"o",
"w"
],
[
"ti",
"c"
],
[
"Ġc",
"on"
],
[
"q",
"u"
],
[
"Ġ",
"h"
],
[
"p",
"er"
],
[
"Ġ",
"on"
],
[
"i",
"g"
],
[
"a",
"m"
],
[
"re",
"s"
],
[
"Ġw",
"ith"
],
[
"Ġth",
"at"
],
[
"Ġ",
"W"
],
[
"v",
"er"
],
[
"u",
"m"
],
[
"Ġ",
"$"
],
[
"i",
"l"
],
[
"Ġe",
"x"
],
[
"u",
"t"
],
[
"s",
"e"
],
[
"o",
"t"
],
[
"at",
"e"
],
[
"e",
"c"
],
[
"i",
"d"
],
[
"e",
"m"
],
[
"it",
"y"
],
[
"Ġw",
"e"
],
[
"p",
"l"
],
[
"ĠW",
"e"
],
[
"i",
"f"
],
[
"Ġ",
"("
],
[
"a",
"b"
],
[
"g",
"e"
],
[
"ĠT",
"he"
],
[
"r",
"i"
],
[
"t",
"er"
],
[
"t",
"he"
],
[
"Ġa",
"l"
],
[
"Ġa",
"s"
],
[
"Ġb",
"e"
],
[
"Ġa",
"re"
],
[
"e",
"ct"
],
[
"om",
"p"
],
[
"c",
"tion"
],
[
"ation",
"s"
],
[
"u",
"re"
],
[
"Ġ",
"I"
],
[
"Ġm",
"od"
],
[
"Ġ",
"v"
],
[
"i",
"r"
],
[
"o",
"s"
],
[
"Ġs",
"u"
],
[
"Ġb",
"y"
],
[
"t",
"s"
],
[
"Ġs",
"p"
],
[
"p",
"ro"
],
[
"un",
"d"
],
[
"i",
"z"
],
[
"a",
"c"
],
[
"ic",
"al"
],
[
"f",
"or"
],
[
"r",
"an"
],
[
"an",
"t"
],
[
"Ġc",
"omp"
],
[
"d",
"i"
],
[
"a",
"in"
],
[
"Ġw",
"h"
],
[
"en",
"er"
],
[
"re",
"n"
],
[
"ro",
"m"
],
[
"Ġmod",
"el"
],
[
"Ġ",
"us"
],
[
"Ġ",
"A"
],
[
"Ġth",
"is"
],
[
"is",
"t"
],
[
"ti",
"ve"
],
[
"Ġre",
"s"
],
[
"si",
"on"
],
[
"a",
"g"
],
[
"Ġd",
"e"
],
[
"d",
"u"
],
[
"Ġs",
"h"
],
[
"es",
"s"
],
[
"Ġa",
"t"
],
[
"Ġ",
"S"
],
[
"o",
"c"
]
]
}
}

41
tiny_llm_pretrain/checkpoint-9900/tokenizer_config.json Normal file
View File

@@ -0,0 +1,41 @@
{
"added_tokens_decoder": {
"0": {
"content": "<|endoftext|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"1": {
"content": "<|im_start|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"2": {
"content": "<|im_end|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
}
},
"bos_token": "<|endoftext|>",
"chat_template": "{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
"clean_up_tokenization_spaces": false,
"eos_token": "<|endoftext|>",
"extra_special_tokens": {},
"model_input_names": [
"input_ids",
"attention_mask"
],
"model_max_length": 1000000000000000019884624838656,
"pad_token": "<|endoftext|>",
"tokenizer_class": "PreTrainedTokenizerFast",
"unk_token": "<|endoftext|>"
}

13893
tiny_llm_pretrain/checkpoint-9900/trainer_state.json Normal file
View File

File diff suppressed because it is too large Load Diff

3
tiny_llm_pretrain/checkpoint-9900/training_args.bin Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:3094fcb2cbf0c2dbb4dd559e2ad79c9e24d99aeecd652bbb12aa828bc03d3da2
size 5713

951
tokenizer.json Normal file
View File

@@ -0,0 +1,951 @@
{
"version": "1.0",
"truncation": null,
"padding": null,
"added_tokens": [
{
"id": 0,
"content": "<|endoftext|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 1,
"content": "<|im_start|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 2,
"content": "<|im_end|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
}
],
"normalizer": null,
"pre_tokenizer": {
"type": "ByteLevel",
"add_prefix_space": false,
"trim_offsets": true,
"use_regex": true
},
"post_processor": {
"type": "ByteLevel",
"add_prefix_space": true,
"trim_offsets": false,
"use_regex": true
},
"decoder": {
"type": "ByteLevel",
"add_prefix_space": true,
"trim_offsets": true,
"use_regex": true
},
"model": {
"type": "BPE",
"dropout": null,
"unk_token": null,
"continuing_subword_prefix": null,
"end_of_word_suffix": null,
"fuse_unk": false,
"byte_fallback": false,
"ignore_merges": false,
"vocab": {
"<|endoftext|>": 0,
"<|im_start|>": 1,
"<|im_end|>": 2,
"!": 3,
"\"": 4,
"#": 5,
"$": 6,
"%": 7,
"&": 8,
"'": 9,
"(": 10,
")": 11,
"*": 12,
"+": 13,
",": 14,
"-": 15,
".": 16,
"/": 17,
"0": 18,
"1": 19,
"2": 20,
"3": 21,
"4": 22,
"5": 23,
"6": 24,
"7": 25,
"8": 26,
"9": 27,
":": 28,
";": 29,
"<": 30,
"=": 31,
">": 32,
"?": 33,
"@": 34,
"A": 35,
"B": 36,
"C": 37,
"D": 38,
"E": 39,
"F": 40,
"G": 41,
"H": 42,
"I": 43,
"J": 44,
"K": 45,
"L": 46,
"M": 47,
"N": 48,
"O": 49,
"P": 50,
"Q": 51,
"R": 52,
"S": 53,
"T": 54,
"U": 55,
"V": 56,
"W": 57,
"X": 58,
"Y": 59,
"Z": 60,
"[": 61,
"\\": 62,
"]": 63,
"^": 64,
"_": 65,
"`": 66,
"a": 67,
"b": 68,
"c": 69,
"d": 70,
"e": 71,
"f": 72,
"g": 73,
"h": 74,
"i": 75,
"j": 76,
"k": 77,
"l": 78,
"m": 79,
"n": 80,
"o": 81,
"p": 82,
"q": 83,
"r": 84,
"s": 85,
"t": 86,
"u": 87,
"v": 88,
"w": 89,
"x": 90,
"y": 91,
"z": 92,
"{": 93,
"|": 94,
"}": 95,
"~": 96,
"Ċ": 97,
"Ġ": 98,
"Ġt": 99,
"Ġa": 100,
"in": 101,
"on": 102,
"he": 103,
"ti": 104,
"re": 105,
"Ġthe": 106,
"er": 107,
"Ġs": 108,
"Ġo": 109,
"en": 110,
"al": 111,
"Ġc": 112,
"or": 113,
"at": 114,
"tion": 115,
"es": 116,
"Ġof": 117,
"is": 118,
"Ġp": 119,
"an": 120,
"ed": 121,
"Ġf": 122,
"ar": 123,
"it": 124,
"ro": 125,
"ing": 126,
"Ġan": 127,
"Ġin": 128,
"Ġd": 129,
"Ġw": 130,
"Ġm": 131,
"ic": 132,
"le": 133,
"Ġb": 134,
"Ġand": 135,
"ation": 136,
"as": 137,
"Ġe": 138,
"el": 139,
"Ġto": 140,
"et": 141,
"ent": 142,
"Ġre": 143,
"st": 144,
"ct": 145,
"Ġth": 146,
"om": 147,
"im": 148,
"ul": 149,
"ra": 150,
"Ġn": 151,
"ve": 152,
"ly": 153,
"od": 154,
"ur": 155,
"un": 156,
"si": 157,
"ce": 158,
"Ġis": 159,
"Ġl": 160,
"Ġfor": 161,
"us": 162,
"ith": 163,
"ĠT": 164,
"ol": 165,
"Ġg": 166,
"Ġst": 167,
"Ġpro": 168,
"ch": 169,
"ow": 170,
"tic": 171,
"Ġcon": 172,
"qu": 173,
"Ġh": 174,
"per": 175,
"Ġon": 176,
"ig": 177,
"am": 178,
"res": 179,
"Ġwith": 180,
"Ġthat": 181,
"ĠW": 182,
"ver": 183,
"um": 184,
"Ġ$": 185,
"il": 186,
"Ġex": 187,
"ut": 188,
"se": 189,
"ot": 190,
"ate": 191,
"ec": 192,
"id": 193,
"em": 194,
"ity": 195,
"Ġwe": 196,
"pl": 197,
"ĠWe": 198,
"if": 199,
"Ġ(": 200,
"ab": 201,
"ge": 202,
"ĠThe": 203,
"ri": 204,
"ter": 205,
"the": 206,
"Ġal": 207,
"Ġas": 208,
"Ġbe": 209,
"Ġare": 210,
"ect": 211,
"omp": 212,
"ction": 213,
"ations": 214,
"ure": 215,
"ĠI": 216,
"Ġmod": 217,
"Ġv": 218,
"ir": 219,
"os": 220,
"Ġsu": 221,
"Ġby": 222,
"ts": 223,
"Ġsp": 224,
"pro": 225,
"und": 226,
"iz": 227,
"ac": 228,
"ical": 229,
"for": 230,
"ran": 231,
"ant": 232,
"Ġcomp": 233,
"di": 234,
"ain": 235,
"Ġwh": 236,
"ener": 237,
"ren": 238,
"rom": 239,
"Ġmodel": 240,
"Ġus": 241,
"ĠA": 242,
"Ġthis": 243,
"ist": 244,
"tive": 245,
"Ġres": 246,
"sion": 247,
"ag": 248,
"Ġde": 249,
"du": 250,
"Ġsh": 251,
"ess": 252,
"Ġat": 253,
"ĠS": 254,
"oc": 255
},
"merges": [
[
"Ġ",
"t"
],
[
"Ġ",
"a"
],
[
"i",
"n"
],
[
"o",
"n"
],
[
"h",
"e"
],
[
"t",
"i"
],
[
"r",
"e"
],
[
"Ġt",
"he"
],
[
"e",
"r"
],
[
"Ġ",
"s"
],
[
"Ġ",
"o"
],
[
"e",
"n"
],
[
"a",
"l"
],
[
"Ġ",
"c"
],
[
"o",
"r"
],
[
"a",
"t"
],
[
"ti",
"on"
],
[
"e",
"s"
],
[
"Ġo",
"f"
],
[
"i",
"s"
],
[
"Ġ",
"p"
],
[
"a",
"n"
],
[
"e",
"d"
],
[
"Ġ",
"f"
],
[
"a",
"r"
],
[
"i",
"t"
],
[
"r",
"o"
],
[
"in",
"g"
],
[
"Ġa",
"n"
],
[
"Ġ",
"in"
],
[
"Ġ",
"d"
],
[
"Ġ",
"w"
],
[
"Ġ",
"m"
],
[
"i",
"c"
],
[
"l",
"e"
],
[
"Ġ",
"b"
],
[
"Ġan",
"d"
],
[
"a",
"tion"
],
[
"a",
"s"
],
[
"Ġ",
"e"
],
[
"e",
"l"
],
[
"Ġt",
"o"
],
[
"e",
"t"
],
[
"en",
"t"
],
[
"Ġ",
"re"
],
[
"s",
"t"
],
[
"c",
"t"
],
[
"Ġt",
"h"
],
[
"o",
"m"
],
[
"i",
"m"
],
[
"u",
"l"
],
[
"r",
"a"
],
[
"Ġ",
"n"
],
[
"v",
"e"
],
[
"l",
"y"
],
[
"o",
"d"
],
[
"u",
"r"
],
[
"u",
"n"
],
[
"s",
"i"
],
[
"c",
"e"
],
[
"Ġ",
"is"
],
[
"Ġ",
"l"
],
[
"Ġf",
"or"
],
[
"u",
"s"
],
[
"it",
"h"
],
[
"Ġ",
"T"
],
[
"o",
"l"
],
[
"Ġ",
"g"
],
[
"Ġs",
"t"
],
[
"Ġp",
"ro"
],
[
"c",
"h"
],
[
"o",
"w"
],
[
"ti",
"c"
],
[
"Ġc",
"on"
],
[
"q",
"u"
],
[
"Ġ",
"h"
],
[
"p",
"er"
],
[
"Ġ",
"on"
],
[
"i",
"g"
],
[
"a",
"m"
],
[
"re",
"s"
],
[
"Ġw",
"ith"
],
[
"Ġth",
"at"
],
[
"Ġ",
"W"
],
[
"v",
"er"
],
[
"u",
"m"
],
[
"Ġ",
"$"
],
[
"i",
"l"
],
[
"Ġe",
"x"
],
[
"u",
"t"
],
[
"s",
"e"
],
[
"o",
"t"
],
[
"at",
"e"
],
[
"e",
"c"
],
[
"i",
"d"
],
[
"e",
"m"
],
[
"it",
"y"
],
[
"Ġw",
"e"
],
[
"p",
"l"
],
[
"ĠW",
"e"
],
[
"i",
"f"
],
[
"Ġ",
"("
],
[
"a",
"b"
],
[
"g",
"e"
],
[
"ĠT",
"he"
],
[
"r",
"i"
],
[
"t",
"er"
],
[
"t",
"he"
],
[
"Ġa",
"l"
],
[
"Ġa",
"s"
],
[
"Ġb",
"e"
],
[
"Ġa",
"re"
],
[
"e",
"ct"
],
[
"om",
"p"
],
[
"c",
"tion"
],
[
"ation",
"s"
],
[
"u",
"re"
],
[
"Ġ",
"I"
],
[
"Ġm",
"od"
],
[
"Ġ",
"v"
],
[
"i",
"r"
],
[
"o",
"s"
],
[
"Ġs",
"u"
],
[
"Ġb",
"y"
],
[
"t",
"s"
],
[
"Ġs",
"p"
],
[
"p",
"ro"
],
[
"un",
"d"
],
[
"i",
"z"
],
[
"a",
"c"
],
[
"ic",
"al"
],
[
"f",
"or"
],
[
"r",
"an"
],
[
"an",
"t"
],
[
"Ġc",
"omp"
],
[
"d",
"i"
],
[
"a",
"in"
],
[
"Ġw",
"h"
],
[
"en",
"er"
],
[
"re",
"n"
],
[
"ro",
"m"
],
[
"Ġmod",
"el"
],
[
"Ġ",
"us"
],
[
"Ġ",
"A"
],
[
"Ġth",
"is"
],
[
"is",
"t"
],
[
"ti",
"ve"
],
[
"Ġre",
"s"
],
[
"si",
"on"
],
[
"a",
"g"
],
[
"Ġd",
"e"
],
[
"d",
"u"
],
[
"Ġs",
"h"
],
[
"es",
"s"
],
[
"Ġa",
"t"
],
[
"Ġ",
"S"
],
[
"o",
"c"
]
]
}
}

41
tokenizer_config.json Normal file
View File

@@ -0,0 +1,41 @@
{
"added_tokens_decoder": {
"0": {
"content": "<|endoftext|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"1": {
"content": "<|im_start|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"2": {
"content": "<|im_end|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
}
},
"bos_token": "<|endoftext|>",
"chat_template": "{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
"clean_up_tokenization_spaces": false,
"eos_token": "<|endoftext|>",
"extra_special_tokens": {},
"model_input_names": [
"input_ids",
"attention_mask"
],
"model_max_length": 1000000000000000019884624838656,
"pad_token": "<|endoftext|>",
"tokenizer_class": "PreTrainedTokenizerFast",
"unk_token": "<|endoftext|>"
}

14033
trainer_state.json Normal file
View File

File diff suppressed because it is too large Load Diff

3
training_args.bin Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:3094fcb2cbf0c2dbb4dd559e2ad79c9e24d99aeecd652bbb12aa828bc03d3da2
size 5713