1039 lines
28 KiB
JSON
1039 lines
28 KiB
JSON
{
|
|
"added_tokens_decoder": {
|
|
"0": {
|
|
"content": "<[begin\u2581of\u2581sentence]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"1": {
|
|
"content": "<[PAD\u2581TOKEN]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"2": {
|
|
"content": "<[end\u2581of\u2581sentence]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"3": {
|
|
"content": "<[UNK_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"4": {
|
|
"content": "<[CLS_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"5": {
|
|
"content": "<[MASK_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"6": {
|
|
"content": "<[SEP\u2581TOKEN]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"7": {
|
|
"content": "<[PLHD7_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"8": {
|
|
"content": "<[PLHD8_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"9": {
|
|
"content": "<[PLHD9_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"10": {
|
|
"content": "<[PLHD10_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"11": {
|
|
"content": "<[PLHD11_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"12": {
|
|
"content": "<[PLHD12_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"13": {
|
|
"content": "<[PLHD13_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"14": {
|
|
"content": "<[PLHD14_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"15": {
|
|
"content": "<[PLHD15_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"16": {
|
|
"content": "<[PLHD16_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"17": {
|
|
"content": "<[PLHD17_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"18": {
|
|
"content": "<[PLHD18_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"19": {
|
|
"content": "<[PLHD19_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"20": {
|
|
"content": "<[PLHD20_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"21": {
|
|
"content": "<[PLHD21_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"22": {
|
|
"content": "<[PLHD22_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"23": {
|
|
"content": "<[PLHD23_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"24": {
|
|
"content": "<[PLHD24_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"25": {
|
|
"content": "<[PLHD25_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"26": {
|
|
"content": "<[PLHD26_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"27": {
|
|
"content": "<[PLHD27_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"28": {
|
|
"content": "<[PLHD28_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"29": {
|
|
"content": "<[PLHD29_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"30": {
|
|
"content": "<[PLHD30_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"31": {
|
|
"content": "<[PLHD31_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32": {
|
|
"content": "<[PLHD32_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"33": {
|
|
"content": "<[PLHD33_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"34": {
|
|
"content": "<[PLHD34_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"35": {
|
|
"content": "<[PLHD35_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"36": {
|
|
"content": "<[PLHD36_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"37": {
|
|
"content": "<[PLHD37_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"38": {
|
|
"content": "<[PLHD38_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"39": {
|
|
"content": "<[PLHD39_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"40": {
|
|
"content": "<[PLHD40_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"41": {
|
|
"content": "<[PLHD41_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"42": {
|
|
"content": "<[PLHD42_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"43": {
|
|
"content": "<[PLHD43_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"44": {
|
|
"content": "<[PLHD44_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"45": {
|
|
"content": "<[PLHD45_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"46": {
|
|
"content": "<[PLHD46_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"47": {
|
|
"content": "<[PLHD47_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"48": {
|
|
"content": "<[PLHD48_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"49": {
|
|
"content": "<[PLHD49_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"50": {
|
|
"content": "<[PLHD50_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"51": {
|
|
"content": "<[PLHD51_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"52": {
|
|
"content": "<[PLHD52_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"53": {
|
|
"content": "<[PLHD53_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"54": {
|
|
"content": "<[PLHD54_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"55": {
|
|
"content": "<[PLHD55_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"56": {
|
|
"content": "<[PLHD56_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"57": {
|
|
"content": "<[PLHD57_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"58": {
|
|
"content": "<[PLHD58_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"59": {
|
|
"content": "<[PLHD59_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"60": {
|
|
"content": "<[PLHD60_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"61": {
|
|
"content": "<[PLHD61_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"62": {
|
|
"content": "<[PLHD62_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"63": {
|
|
"content": "<[PLHD63_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"64": {
|
|
"content": "<[PLHD64_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"65": {
|
|
"content": "<[PLHD65_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"66": {
|
|
"content": "<[PLHD66_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"67": {
|
|
"content": "<[PLHD67_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"68": {
|
|
"content": "<[PLHD68_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"69": {
|
|
"content": "<[PLHD69_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"70": {
|
|
"content": "<[PLHD70_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"71": {
|
|
"content": "<[PLHD71_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"72": {
|
|
"content": "<[PLHD72_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"73": {
|
|
"content": "<[PLHD73_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"74": {
|
|
"content": "<[PLHD74_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"75": {
|
|
"content": "<[PLHD75_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"76": {
|
|
"content": "<[PLHD76_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"77": {
|
|
"content": "<[PLHD77_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"78": {
|
|
"content": "<[PLHD78_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"79": {
|
|
"content": "<[PLHD79_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"80": {
|
|
"content": "<[PLHD80_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"81": {
|
|
"content": "<[PLHD81_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"82": {
|
|
"content": "<[PLHD82_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"83": {
|
|
"content": "<[PLHD83_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"84": {
|
|
"content": "<[PLHD84_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"85": {
|
|
"content": "<[PLHD85_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"86": {
|
|
"content": "<[PLHD86_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"87": {
|
|
"content": "<[PLHD87_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"88": {
|
|
"content": "<[PLHD88_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"89": {
|
|
"content": "<[PLHD89_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"90": {
|
|
"content": "<[PLHD90_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"91": {
|
|
"content": "<[PLHD91_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"92": {
|
|
"content": "<[PLHD92_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"93": {
|
|
"content": "<[PLHD93_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"94": {
|
|
"content": "<[PLHD94_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"95": {
|
|
"content": "<[PLHD95_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"96": {
|
|
"content": "<[PLHD96_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"97": {
|
|
"content": "<[PLHD97_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"98": {
|
|
"content": "<[PLHD98_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"99": {
|
|
"content": "<[PLHD99_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"100": {
|
|
"content": "<[PLHD100_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"101": {
|
|
"content": "<[PLHD101_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"102": {
|
|
"content": "<[PLHD102_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"103": {
|
|
"content": "<[PLHD103_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"104": {
|
|
"content": "<[PLHD104_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"105": {
|
|
"content": "<[PLHD105_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"106": {
|
|
"content": "<[PLHD106_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"107": {
|
|
"content": "<[PLHD107_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"108": {
|
|
"content": "<[PLHD108_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"109": {
|
|
"content": "<[PLHD109_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"110": {
|
|
"content": "<[PLHD110_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"111": {
|
|
"content": "<[PLHD111_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"112": {
|
|
"content": "<[PLHD112_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"113": {
|
|
"content": "<[PLHD113_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"114": {
|
|
"content": "<[PLHD114_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"115": {
|
|
"content": "<[PLHD115_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"116": {
|
|
"content": "<[PLHD116_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"117": {
|
|
"content": "<[PLHD117_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"118": {
|
|
"content": "<[PLHD118_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"119": {
|
|
"content": "<[PLHD119_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"120": {
|
|
"content": "<[PLHD120_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"121": {
|
|
"content": "<[PLHD121_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"122": {
|
|
"content": "<[PLHD122_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"123": {
|
|
"content": "<[PLHD123_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"124": {
|
|
"content": "<[fim-prefix]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"125": {
|
|
"content": "<[fim-suffix]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"126": {
|
|
"content": "<[fim-middle]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"127": {
|
|
"content": "<[PLHD127_never_used_51bce0c785ca2f68081bfa7d91973934]>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
}
|
|
},
|
|
"bos_token": "<[begin\u2581of\u2581sentence]>",
|
|
"chat_template": "{% if messages[0]['role'] == 'system' %}{{ raise_exception('System role not supported') }}{% endif %}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% set role = message['role'] %}{{ bos_token + role + '\n' + message['content'] | trim + eos_token }}{% endfor %}{% if add_generation_prompt %}{{ bos_token + 'assistant\n'}}{% endif %}",
|
|
"clean_up_tokenization_spaces": false,
|
|
"eos_token": "<[end\u2581of\u2581sentence]>",
|
|
"extra_special_tokens": {},
|
|
"model_max_length": 65536,
|
|
"pad_token": "<[PAD\u2581TOKEN]>",
|
|
"padding_side": "left",
|
|
"sep_token": "<[SEP\u2581TOKEN]>",
|
|
"tokenizer_class": "PreTrainedTokenizer",
|
|
"unk_token": null
|
|
} |