714 lines
9.2 KiB
JSON
714 lines
9.2 KiB
JSON
{
|
|
"version": "1.0",
|
|
"truncation": null,
|
|
"padding": null,
|
|
"added_tokens": [
|
|
{
|
|
"id": 0,
|
|
"content": "<|endoftext|>",
|
|
"single_word": false,
|
|
"lstrip": false,
|
|
"rstrip": false,
|
|
"normalized": false,
|
|
"special": true
|
|
},
|
|
{
|
|
"id": 1,
|
|
"content": "<|unk|>",
|
|
"single_word": false,
|
|
"lstrip": false,
|
|
"rstrip": false,
|
|
"normalized": false,
|
|
"special": true
|
|
}
|
|
],
|
|
"normalizer": null,
|
|
"pre_tokenizer": {
|
|
"type": "ByteLevel",
|
|
"add_prefix_space": false,
|
|
"trim_offsets": true,
|
|
"use_regex": true
|
|
},
|
|
"post_processor": null,
|
|
"decoder": {
|
|
"type": "ByteLevel",
|
|
"add_prefix_space": true,
|
|
"trim_offsets": true,
|
|
"use_regex": true
|
|
},
|
|
"model": {
|
|
"type": "BPE",
|
|
"dropout": null,
|
|
"unk_token": "<|unk|>",
|
|
"continuing_subword_prefix": null,
|
|
"end_of_word_suffix": null,
|
|
"fuse_unk": false,
|
|
"byte_fallback": false,
|
|
"ignore_merges": false,
|
|
"vocab": {
|
|
"<|endoftext|>": 0,
|
|
"<|unk|>": 1,
|
|
"!": 2,
|
|
".": 3,
|
|
"H": 4,
|
|
"I": 5,
|
|
"O": 6,
|
|
"T": 7,
|
|
"a": 8,
|
|
"b": 9,
|
|
"c": 10,
|
|
"d": 11,
|
|
"e": 12,
|
|
"f": 13,
|
|
"g": 14,
|
|
"h": 15,
|
|
"i": 16,
|
|
"j": 17,
|
|
"k": 18,
|
|
"l": 19,
|
|
"m": 20,
|
|
"n": 21,
|
|
"o": 22,
|
|
"p": 23,
|
|
"q": 24,
|
|
"r": 25,
|
|
"s": 26,
|
|
"t": 27,
|
|
"u": 28,
|
|
"v": 29,
|
|
"w": 30,
|
|
"x": 31,
|
|
"y": 32,
|
|
"z": 33,
|
|
"Ċ": 34,
|
|
"Ġ": 35,
|
|
"Ġt": 36,
|
|
"Ġa": 37,
|
|
"el": 38,
|
|
"er": 39,
|
|
"Th": 40,
|
|
"del": 41,
|
|
"es": 42,
|
|
"fo": 43,
|
|
"in": 44,
|
|
"is": 45,
|
|
"la": 46,
|
|
"mo": 47,
|
|
"Ġla": 48,
|
|
"Ġmo": 49,
|
|
"Ġtes": 50,
|
|
"Ġmodel": 51,
|
|
"Ġtest": 52,
|
|
"al": 53,
|
|
"as": 54,
|
|
"ed": 55,
|
|
"on": 56,
|
|
"ra": 57,
|
|
"wo": 58,
|
|
"Ġb": 59,
|
|
"Ġo": 60,
|
|
"Ġu": 61,
|
|
"Ġfo": 62,
|
|
"Ġis": 63,
|
|
"Ġth": 64,
|
|
"Ġtin": 65,
|
|
"The": 66,
|
|
"Ġtiny": 67,
|
|
"Hel": 68,
|
|
"In": 69,
|
|
"On": 70,
|
|
"ag": 71,
|
|
"ce": 72,
|
|
"ck": 73,
|
|
"do": 74,
|
|
"eg": 75,
|
|
"eed": 76,
|
|
"gu": 77,
|
|
"has": 78,
|
|
"id": 79,
|
|
"im": 80,
|
|
"ion": 81,
|
|
"ick": 82,
|
|
"ju": 83,
|
|
"ld": 84,
|
|
"lo": 85,
|
|
"mp": 86,
|
|
"mer": 87,
|
|
"mal": 88,
|
|
"nd": 89,
|
|
"ns": 90,
|
|
"need": 91,
|
|
"ngu": 92,
|
|
"ow": 93,
|
|
"pon": 94,
|
|
"qu": 95,
|
|
"rld": 96,
|
|
"rmer": 97,
|
|
"row": 98,
|
|
"sed": 99,
|
|
"smal": 100,
|
|
"teg": 101,
|
|
"tion": 102,
|
|
"ut": 103,
|
|
"ver": 104,
|
|
"val": 105,
|
|
"was": 106,
|
|
"yer": 107,
|
|
"zy": 108,
|
|
"ĠTh": 109,
|
|
"Ġwo": 110,
|
|
"Ġdo": 111,
|
|
"Ġhas": 112,
|
|
"Ġju": 113,
|
|
"Ġneed": 114,
|
|
"Ġqu": 115,
|
|
"Ġsmal": 116,
|
|
"Ġval": 117,
|
|
"Ġwas": 118,
|
|
"Ġtra": 119,
|
|
"Ġtwo": 120,
|
|
"Ġtim": 121,
|
|
"Ġand": 122,
|
|
"ere": 123,
|
|
"former": 124,
|
|
"ing": 125,
|
|
"Ġlangu": 126,
|
|
"Ġlayer": 127,
|
|
"Ġlazy": 128,
|
|
"Ġtests": 129,
|
|
"Ġtesting": 130,
|
|
"ration": 131,
|
|
"Ġbrow": 132,
|
|
"Ġbut": 133,
|
|
"Ġof": 134,
|
|
"Ġover": 135,
|
|
"Ġupon": 136,
|
|
"Ġused": 137,
|
|
"Ġfor": 138,
|
|
"Ġfox": 139,
|
|
"Ġthe": 140,
|
|
"Ġthere": 141,
|
|
"Hello": 142,
|
|
"Integ": 143,
|
|
"Once": 144,
|
|
"age": 145,
|
|
"mps": 146,
|
|
"nsformer": 147,
|
|
"ĠThis": 148,
|
|
"Ġworld": 149,
|
|
"Ġdog": 150,
|
|
"Ġjumps": 151,
|
|
"Ġquick": 152,
|
|
"Ġsmall": 153,
|
|
"Ġvalid": 154,
|
|
"Ġtransformer": 155,
|
|
"Ġtime": 156,
|
|
"Ġlanguage": 157,
|
|
"Ġlayers": 158,
|
|
"Ġbrown": 159,
|
|
"Integration": 160
|
|
},
|
|
"merges": [
|
|
[
|
|
"Ġ",
|
|
"t"
|
|
],
|
|
[
|
|
"Ġ",
|
|
"a"
|
|
],
|
|
[
|
|
"e",
|
|
"l"
|
|
],
|
|
[
|
|
"e",
|
|
"r"
|
|
],
|
|
[
|
|
"T",
|
|
"h"
|
|
],
|
|
[
|
|
"d",
|
|
"el"
|
|
],
|
|
[
|
|
"e",
|
|
"s"
|
|
],
|
|
[
|
|
"f",
|
|
"o"
|
|
],
|
|
[
|
|
"i",
|
|
"n"
|
|
],
|
|
[
|
|
"i",
|
|
"s"
|
|
],
|
|
[
|
|
"l",
|
|
"a"
|
|
],
|
|
[
|
|
"m",
|
|
"o"
|
|
],
|
|
[
|
|
"Ġ",
|
|
"la"
|
|
],
|
|
[
|
|
"Ġ",
|
|
"mo"
|
|
],
|
|
[
|
|
"Ġt",
|
|
"es"
|
|
],
|
|
[
|
|
"Ġmo",
|
|
"del"
|
|
],
|
|
[
|
|
"Ġtes",
|
|
"t"
|
|
],
|
|
[
|
|
"a",
|
|
"l"
|
|
],
|
|
[
|
|
"a",
|
|
"s"
|
|
],
|
|
[
|
|
"e",
|
|
"d"
|
|
],
|
|
[
|
|
"o",
|
|
"n"
|
|
],
|
|
[
|
|
"r",
|
|
"a"
|
|
],
|
|
[
|
|
"w",
|
|
"o"
|
|
],
|
|
[
|
|
"Ġ",
|
|
"b"
|
|
],
|
|
[
|
|
"Ġ",
|
|
"o"
|
|
],
|
|
[
|
|
"Ġ",
|
|
"u"
|
|
],
|
|
[
|
|
"Ġ",
|
|
"fo"
|
|
],
|
|
[
|
|
"Ġ",
|
|
"is"
|
|
],
|
|
[
|
|
"Ġt",
|
|
"h"
|
|
],
|
|
[
|
|
"Ġt",
|
|
"in"
|
|
],
|
|
[
|
|
"Th",
|
|
"e"
|
|
],
|
|
[
|
|
"Ġtin",
|
|
"y"
|
|
],
|
|
[
|
|
"H",
|
|
"el"
|
|
],
|
|
[
|
|
"I",
|
|
"n"
|
|
],
|
|
[
|
|
"O",
|
|
"n"
|
|
],
|
|
[
|
|
"a",
|
|
"g"
|
|
],
|
|
[
|
|
"c",
|
|
"e"
|
|
],
|
|
[
|
|
"c",
|
|
"k"
|
|
],
|
|
[
|
|
"d",
|
|
"o"
|
|
],
|
|
[
|
|
"e",
|
|
"g"
|
|
],
|
|
[
|
|
"e",
|
|
"ed"
|
|
],
|
|
[
|
|
"g",
|
|
"u"
|
|
],
|
|
[
|
|
"h",
|
|
"as"
|
|
],
|
|
[
|
|
"i",
|
|
"d"
|
|
],
|
|
[
|
|
"i",
|
|
"m"
|
|
],
|
|
[
|
|
"i",
|
|
"on"
|
|
],
|
|
[
|
|
"i",
|
|
"ck"
|
|
],
|
|
[
|
|
"j",
|
|
"u"
|
|
],
|
|
[
|
|
"l",
|
|
"d"
|
|
],
|
|
[
|
|
"l",
|
|
"o"
|
|
],
|
|
[
|
|
"m",
|
|
"p"
|
|
],
|
|
[
|
|
"m",
|
|
"er"
|
|
],
|
|
[
|
|
"m",
|
|
"al"
|
|
],
|
|
[
|
|
"n",
|
|
"d"
|
|
],
|
|
[
|
|
"n",
|
|
"s"
|
|
],
|
|
[
|
|
"n",
|
|
"eed"
|
|
],
|
|
[
|
|
"n",
|
|
"gu"
|
|
],
|
|
[
|
|
"o",
|
|
"w"
|
|
],
|
|
[
|
|
"p",
|
|
"on"
|
|
],
|
|
[
|
|
"q",
|
|
"u"
|
|
],
|
|
[
|
|
"r",
|
|
"ld"
|
|
],
|
|
[
|
|
"r",
|
|
"mer"
|
|
],
|
|
[
|
|
"r",
|
|
"ow"
|
|
],
|
|
[
|
|
"s",
|
|
"ed"
|
|
],
|
|
[
|
|
"s",
|
|
"mal"
|
|
],
|
|
[
|
|
"t",
|
|
"eg"
|
|
],
|
|
[
|
|
"t",
|
|
"ion"
|
|
],
|
|
[
|
|
"u",
|
|
"t"
|
|
],
|
|
[
|
|
"v",
|
|
"er"
|
|
],
|
|
[
|
|
"v",
|
|
"al"
|
|
],
|
|
[
|
|
"w",
|
|
"as"
|
|
],
|
|
[
|
|
"y",
|
|
"er"
|
|
],
|
|
[
|
|
"z",
|
|
"y"
|
|
],
|
|
[
|
|
"Ġ",
|
|
"Th"
|
|
],
|
|
[
|
|
"Ġ",
|
|
"wo"
|
|
],
|
|
[
|
|
"Ġ",
|
|
"do"
|
|
],
|
|
[
|
|
"Ġ",
|
|
"has"
|
|
],
|
|
[
|
|
"Ġ",
|
|
"ju"
|
|
],
|
|
[
|
|
"Ġ",
|
|
"need"
|
|
],
|
|
[
|
|
"Ġ",
|
|
"qu"
|
|
],
|
|
[
|
|
"Ġ",
|
|
"smal"
|
|
],
|
|
[
|
|
"Ġ",
|
|
"val"
|
|
],
|
|
[
|
|
"Ġ",
|
|
"was"
|
|
],
|
|
[
|
|
"Ġt",
|
|
"ra"
|
|
],
|
|
[
|
|
"Ġt",
|
|
"wo"
|
|
],
|
|
[
|
|
"Ġt",
|
|
"im"
|
|
],
|
|
[
|
|
"Ġa",
|
|
"nd"
|
|
],
|
|
[
|
|
"er",
|
|
"e"
|
|
],
|
|
[
|
|
"fo",
|
|
"rmer"
|
|
],
|
|
[
|
|
"in",
|
|
"g"
|
|
],
|
|
[
|
|
"Ġla",
|
|
"ngu"
|
|
],
|
|
[
|
|
"Ġla",
|
|
"yer"
|
|
],
|
|
[
|
|
"Ġla",
|
|
"zy"
|
|
],
|
|
[
|
|
"Ġtest",
|
|
"s"
|
|
],
|
|
[
|
|
"Ġtest",
|
|
"ing"
|
|
],
|
|
[
|
|
"ra",
|
|
"tion"
|
|
],
|
|
[
|
|
"Ġb",
|
|
"row"
|
|
],
|
|
[
|
|
"Ġb",
|
|
"ut"
|
|
],
|
|
[
|
|
"Ġo",
|
|
"f"
|
|
],
|
|
[
|
|
"Ġo",
|
|
"ver"
|
|
],
|
|
[
|
|
"Ġu",
|
|
"pon"
|
|
],
|
|
[
|
|
"Ġu",
|
|
"sed"
|
|
],
|
|
[
|
|
"Ġfo",
|
|
"r"
|
|
],
|
|
[
|
|
"Ġfo",
|
|
"x"
|
|
],
|
|
[
|
|
"Ġth",
|
|
"e"
|
|
],
|
|
[
|
|
"Ġth",
|
|
"ere"
|
|
],
|
|
[
|
|
"Hel",
|
|
"lo"
|
|
],
|
|
[
|
|
"In",
|
|
"teg"
|
|
],
|
|
[
|
|
"On",
|
|
"ce"
|
|
],
|
|
[
|
|
"ag",
|
|
"e"
|
|
],
|
|
[
|
|
"mp",
|
|
"s"
|
|
],
|
|
[
|
|
"ns",
|
|
"former"
|
|
],
|
|
[
|
|
"ĠTh",
|
|
"is"
|
|
],
|
|
[
|
|
"Ġwo",
|
|
"rld"
|
|
],
|
|
[
|
|
"Ġdo",
|
|
"g"
|
|
],
|
|
[
|
|
"Ġju",
|
|
"mps"
|
|
],
|
|
[
|
|
"Ġqu",
|
|
"ick"
|
|
],
|
|
[
|
|
"Ġsmal",
|
|
"l"
|
|
],
|
|
[
|
|
"Ġval",
|
|
"id"
|
|
],
|
|
[
|
|
"Ġtra",
|
|
"nsformer"
|
|
],
|
|
[
|
|
"Ġtim",
|
|
"e"
|
|
],
|
|
[
|
|
"Ġlangu",
|
|
"age"
|
|
],
|
|
[
|
|
"Ġlayer",
|
|
"s"
|
|
],
|
|
[
|
|
"Ġbrow",
|
|
"n"
|
|
],
|
|
[
|
|
"Integ",
|
|
"ration"
|
|
]
|
|
]
|
|
}
|
|
} |