1068 lines
24 KiB
JSON
1068 lines
24 KiB
JSON
{
|
|
"add_bos_token": true,
|
|
"add_eos_token": false,
|
|
"add_prefix_space": null,
|
|
"added_tokens_decoder": {
|
|
"0": {
|
|
"content": "<unk>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"1": {
|
|
"content": "<|startoftext|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"2": {
|
|
"content": "<|endoftext|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32000": {
|
|
"content": "<|end|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32001": {
|
|
"content": "<|assistant|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32002": {
|
|
"content": "<|placeholder1|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32003": {
|
|
"content": "<|placeholder2|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32004": {
|
|
"content": "<|placeholder3|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32005": {
|
|
"content": "<|placeholder4|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32006": {
|
|
"content": "<|system|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32007": {
|
|
"content": "<|im_end|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32008": {
|
|
"content": "<|placeholder5|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32009": {
|
|
"content": "<|placeholder6|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32010": {
|
|
"content": "<|im_start|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32011": {
|
|
"content": "<|placeholder7|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32012": {
|
|
"content": "<|placeholder8|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32013": {
|
|
"content": "<|placeholder9|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32014": {
|
|
"content": "<|placeholder10|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32015": {
|
|
"content": "<|placeholder11|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32016": {
|
|
"content": "<|placeholder12|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32017": {
|
|
"content": "<|placeholder13|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32018": {
|
|
"content": "<|placeholder14|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32019": {
|
|
"content": "<|placeholder15|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32020": {
|
|
"content": "<|placeholder16|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32021": {
|
|
"content": "<|placeholder17|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32022": {
|
|
"content": "<|placeholder18|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32023": {
|
|
"content": "<|placeholder19|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32024": {
|
|
"content": "<|placeholder20|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32025": {
|
|
"content": "<|placeholder21|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32026": {
|
|
"content": "<|placeholder22|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32027": {
|
|
"content": "<|placeholder23|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32028": {
|
|
"content": "<|placeholder24|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32029": {
|
|
"content": "<|placeholder25|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32030": {
|
|
"content": "<|placeholder26|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32031": {
|
|
"content": "<|placeholder27|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32032": {
|
|
"content": "<|placeholder28|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32033": {
|
|
"content": "<|placeholder29|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32034": {
|
|
"content": "<|placeholder30|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32035": {
|
|
"content": "<|placeholder31|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32036": {
|
|
"content": "<|placeholder32|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32037": {
|
|
"content": "<|placeholder33|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32038": {
|
|
"content": "<|placeholder34|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32039": {
|
|
"content": "<|placeholder35|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32040": {
|
|
"content": "<|placeholder36|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32041": {
|
|
"content": "<|placeholder37|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32042": {
|
|
"content": "<|placeholder38|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32043": {
|
|
"content": "<|placeholder39|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32044": {
|
|
"content": "<|placeholder40|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32045": {
|
|
"content": "<|placeholder41|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32046": {
|
|
"content": "<|placeholder42|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32047": {
|
|
"content": "<|placeholder43|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32048": {
|
|
"content": "<|placeholder44|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32049": {
|
|
"content": "<|placeholder45|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32050": {
|
|
"content": "<|placeholder46|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32051": {
|
|
"content": "<|placeholder47|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32052": {
|
|
"content": "<|placeholder48|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32053": {
|
|
"content": "<|placeholder49|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32054": {
|
|
"content": "<|placeholder50|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32055": {
|
|
"content": "<|placeholder51|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32056": {
|
|
"content": "<|placeholder52|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32057": {
|
|
"content": "<|placeholder53|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32058": {
|
|
"content": "<|placeholder54|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32059": {
|
|
"content": "<|placeholder55|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32060": {
|
|
"content": "<|placeholder56|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32061": {
|
|
"content": "<|placeholder57|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32062": {
|
|
"content": "<|placeholder58|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32063": {
|
|
"content": "<|placeholder59|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32064": {
|
|
"content": "<|placeholder60|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32065": {
|
|
"content": "<|placeholder61|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32066": {
|
|
"content": "<|placeholder62|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32067": {
|
|
"content": "<|placeholder63|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32068": {
|
|
"content": "<|placeholder64|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32069": {
|
|
"content": "<|placeholder65|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32070": {
|
|
"content": "<|placeholder66|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32071": {
|
|
"content": "<|placeholder67|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32072": {
|
|
"content": "<|placeholder68|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32073": {
|
|
"content": "<|placeholder69|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32074": {
|
|
"content": "<|placeholder70|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32075": {
|
|
"content": "<|placeholder71|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32076": {
|
|
"content": "<|placeholder72|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32077": {
|
|
"content": "<|placeholder73|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32078": {
|
|
"content": "<|placeholder74|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32079": {
|
|
"content": "<|placeholder75|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32080": {
|
|
"content": "<|placeholder76|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32081": {
|
|
"content": "<|placeholder77|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32082": {
|
|
"content": "<|placeholder78|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32083": {
|
|
"content": "<|placeholder79|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32084": {
|
|
"content": "<|placeholder80|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32085": {
|
|
"content": "<|placeholder81|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32086": {
|
|
"content": "<|placeholder82|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32087": {
|
|
"content": "<|placeholder83|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32088": {
|
|
"content": "<|placeholder84|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32089": {
|
|
"content": "<|placeholder85|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32090": {
|
|
"content": "<|placeholder86|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32091": {
|
|
"content": "<|placeholder87|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32092": {
|
|
"content": "<|placeholder88|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32093": {
|
|
"content": "<|placeholder89|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32094": {
|
|
"content": "<|placeholder90|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32095": {
|
|
"content": "<|placeholder91|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32096": {
|
|
"content": "<|placeholder92|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32097": {
|
|
"content": "<|placeholder93|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32098": {
|
|
"content": "<|placeholder94|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32099": {
|
|
"content": "<|placeholder95|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32100": {
|
|
"content": "<|placeholder96|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32101": {
|
|
"content": "<|placeholder97|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32102": {
|
|
"content": "<|placeholder98|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32103": {
|
|
"content": "<|placeholder99|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32104": {
|
|
"content": "<|placeholder100|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32105": {
|
|
"content": "<|placeholder101|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32106": {
|
|
"content": "<|placeholder102|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32107": {
|
|
"content": "<|placeholder103|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32108": {
|
|
"content": "<|placeholder104|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32109": {
|
|
"content": "<|placeholder105|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32110": {
|
|
"content": "<|placeholder106|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32111": {
|
|
"content": "<|placeholder107|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32112": {
|
|
"content": "<|placeholder108|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32113": {
|
|
"content": "<|placeholder109|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32114": {
|
|
"content": "<|placeholder110|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32115": {
|
|
"content": "<|placeholder111|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32116": {
|
|
"content": "<|placeholder112|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32117": {
|
|
"content": "<|placeholder113|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32118": {
|
|
"content": "<|placeholder114|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32119": {
|
|
"content": "<|placeholder115|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32120": {
|
|
"content": "<|placeholder116|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32121": {
|
|
"content": "<|placeholder117|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32122": {
|
|
"content": "<|placeholder118|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32123": {
|
|
"content": "<|placeholder119|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32124": {
|
|
"content": "<|placeholder120|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32125": {
|
|
"content": "<|placeholder121|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32126": {
|
|
"content": "<|placeholder122|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32127": {
|
|
"content": "<|placeholder123|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
}
|
|
},
|
|
"bos_token": "<|startoftext|>",
|
|
"chat_template": "{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
|
|
"clean_up_tokenization_spaces": false,
|
|
"eos_token": "<|im_end|>",
|
|
"legacy": true,
|
|
"model_max_length": 4096,
|
|
"pad_token": "<|im_end|>",
|
|
"padding_side": "left",
|
|
"sp_model_kwargs": {},
|
|
"tokenizer_class": "LlamaTokenizer",
|
|
"unk_token": "<unk>",
|
|
"use_default_system_prompt": false
|
|
}
|