322 lines
7.1 KiB
JSON
322 lines
7.1 KiB
JSON
{
|
|
"added_tokens_decoder": {
|
|
"154820": {
|
|
"content": "<|endoftext|>",
|
|
"single_word": false,
|
|
"lstrip": false,
|
|
"rstrip": false,
|
|
"normalized": false,
|
|
"special": true
|
|
},
|
|
"154821": {
|
|
"content": "[MASK]",
|
|
"single_word": false,
|
|
"lstrip": false,
|
|
"rstrip": false,
|
|
"normalized": false,
|
|
"special": true
|
|
},
|
|
"154822": {
|
|
"content": "[gMASK]",
|
|
"single_word": false,
|
|
"lstrip": false,
|
|
"rstrip": false,
|
|
"normalized": false,
|
|
"special": true
|
|
},
|
|
"154823": {
|
|
"content": "[sMASK]",
|
|
"single_word": false,
|
|
"lstrip": false,
|
|
"rstrip": false,
|
|
"normalized": false,
|
|
"special": true
|
|
},
|
|
"154824": {
|
|
"content": "<sop>",
|
|
"single_word": false,
|
|
"lstrip": false,
|
|
"rstrip": false,
|
|
"normalized": false,
|
|
"special": true
|
|
},
|
|
"154825": {
|
|
"content": "<eop>",
|
|
"single_word": false,
|
|
"lstrip": false,
|
|
"rstrip": false,
|
|
"normalized": false,
|
|
"special": true
|
|
},
|
|
"154826": {
|
|
"content": "<|system|>",
|
|
"single_word": false,
|
|
"lstrip": false,
|
|
"rstrip": false,
|
|
"normalized": false,
|
|
"special": true
|
|
},
|
|
"154827": {
|
|
"content": "<|user|>",
|
|
"single_word": false,
|
|
"lstrip": false,
|
|
"rstrip": false,
|
|
"normalized": false,
|
|
"special": true
|
|
},
|
|
"154828": {
|
|
"content": "<|assistant|>",
|
|
"single_word": false,
|
|
"lstrip": false,
|
|
"rstrip": false,
|
|
"normalized": false,
|
|
"special": true
|
|
},
|
|
"154829": {
|
|
"content": "<|observation|>",
|
|
"single_word": false,
|
|
"lstrip": false,
|
|
"rstrip": false,
|
|
"normalized": false,
|
|
"special": true
|
|
},
|
|
"154830": {
|
|
"content": "<|begin_of_image|>",
|
|
"single_word": false,
|
|
"lstrip": false,
|
|
"rstrip": false,
|
|
"normalized": false,
|
|
"special": true
|
|
},
|
|
"154831": {
|
|
"content": "<|end_of_image|>",
|
|
"single_word": false,
|
|
"lstrip": false,
|
|
"rstrip": false,
|
|
"normalized": false,
|
|
"special": true
|
|
},
|
|
"154832": {
|
|
"content": "<|begin_of_video|>",
|
|
"single_word": false,
|
|
"lstrip": false,
|
|
"rstrip": false,
|
|
"normalized": false,
|
|
"special": true
|
|
},
|
|
"154833": {
|
|
"content": "<|end_of_video|>",
|
|
"single_word": false,
|
|
"lstrip": false,
|
|
"rstrip": false,
|
|
"normalized": false,
|
|
"special": true
|
|
},
|
|
"154834": {
|
|
"content": "<|begin_of_audio|>",
|
|
"single_word": false,
|
|
"lstrip": false,
|
|
"rstrip": false,
|
|
"normalized": false,
|
|
"special": true
|
|
},
|
|
"154835": {
|
|
"content": "<|end_of_audio|>",
|
|
"single_word": false,
|
|
"lstrip": false,
|
|
"rstrip": false,
|
|
"normalized": false,
|
|
"special": true
|
|
},
|
|
"154836": {
|
|
"content": "<|begin_of_transcription|>",
|
|
"single_word": false,
|
|
"lstrip": false,
|
|
"rstrip": false,
|
|
"normalized": false,
|
|
"special": true
|
|
},
|
|
"154837": {
|
|
"content": "<|end_of_transcription|>",
|
|
"single_word": false,
|
|
"lstrip": false,
|
|
"rstrip": false,
|
|
"normalized": false,
|
|
"special": true
|
|
},
|
|
"154838": {
|
|
"content": "<|code_prefix|>",
|
|
"single_word": false,
|
|
"lstrip": false,
|
|
"rstrip": false,
|
|
"normalized": false,
|
|
"special": false
|
|
},
|
|
"154839": {
|
|
"content": "<|code_middle|>",
|
|
"single_word": false,
|
|
"lstrip": false,
|
|
"rstrip": false,
|
|
"normalized": false,
|
|
"special": false
|
|
},
|
|
"154840": {
|
|
"content": "<|code_suffix|>",
|
|
"single_word": false,
|
|
"lstrip": false,
|
|
"rstrip": false,
|
|
"normalized": false,
|
|
"special": false
|
|
},
|
|
"154841": {
|
|
"content": "<think>",
|
|
"single_word": false,
|
|
"lstrip": false,
|
|
"rstrip": false,
|
|
"normalized": false,
|
|
"special": false
|
|
},
|
|
"154842": {
|
|
"content": "</think>",
|
|
"single_word": false,
|
|
"lstrip": false,
|
|
"rstrip": false,
|
|
"normalized": false,
|
|
"special": false
|
|
},
|
|
"154843": {
|
|
"content": "<tool_call>",
|
|
"single_word": false,
|
|
"lstrip": false,
|
|
"rstrip": false,
|
|
"normalized": false,
|
|
"special": false
|
|
},
|
|
"154844": {
|
|
"content": "</tool_call>",
|
|
"single_word": false,
|
|
"lstrip": false,
|
|
"rstrip": false,
|
|
"normalized": false,
|
|
"special": false
|
|
},
|
|
"154845": {
|
|
"content": "<tool_response>",
|
|
"single_word": false,
|
|
"lstrip": false,
|
|
"rstrip": false,
|
|
"normalized": false,
|
|
"special": false
|
|
},
|
|
"154846": {
|
|
"content": "</tool_response>",
|
|
"single_word": false,
|
|
"lstrip": false,
|
|
"rstrip": false,
|
|
"normalized": false,
|
|
"special": false
|
|
},
|
|
"154847": {
|
|
"content": "<arg_key>",
|
|
"single_word": false,
|
|
"lstrip": false,
|
|
"rstrip": false,
|
|
"normalized": false,
|
|
"special": false
|
|
},
|
|
"154848": {
|
|
"content": "</arg_key>",
|
|
"single_word": false,
|
|
"lstrip": false,
|
|
"rstrip": false,
|
|
"normalized": false,
|
|
"special": false
|
|
},
|
|
"154849": {
|
|
"content": "<arg_value>",
|
|
"single_word": false,
|
|
"lstrip": false,
|
|
"rstrip": false,
|
|
"normalized": false,
|
|
"special": false
|
|
},
|
|
"154850": {
|
|
"content": "</arg_value>",
|
|
"single_word": false,
|
|
"lstrip": false,
|
|
"rstrip": false,
|
|
"normalized": false,
|
|
"special": false
|
|
},
|
|
"154851": {
|
|
"content": "/nothink",
|
|
"single_word": false,
|
|
"lstrip": false,
|
|
"rstrip": false,
|
|
"normalized": false,
|
|
"special": false
|
|
},
|
|
"154852": {
|
|
"content": "<|begin_of_box|>",
|
|
"single_word": false,
|
|
"lstrip": false,
|
|
"rstrip": false,
|
|
"normalized": false,
|
|
"special": false
|
|
},
|
|
"154853": {
|
|
"content": "<|end_of_box|>",
|
|
"single_word": false,
|
|
"lstrip": false,
|
|
"rstrip": false,
|
|
"normalized": false,
|
|
"special": false
|
|
},
|
|
"154854": {
|
|
"content": "<|image|>",
|
|
"single_word": false,
|
|
"lstrip": false,
|
|
"rstrip": false,
|
|
"normalized": false,
|
|
"special": false
|
|
},
|
|
"154855": {
|
|
"content": "<|video|>",
|
|
"single_word": false,
|
|
"lstrip": false,
|
|
"rstrip": false,
|
|
"normalized": false,
|
|
"special": false
|
|
}
|
|
},
|
|
"additional_special_tokens": [
|
|
"<|endoftext|>",
|
|
"[MASK]",
|
|
"[gMASK]",
|
|
"[sMASK]",
|
|
"<sop>",
|
|
"<eop>",
|
|
"<|system|>",
|
|
"<|user|>",
|
|
"<|assistant|>",
|
|
"<|observation|>",
|
|
"<|begin_of_image|>",
|
|
"<|end_of_image|>",
|
|
"<|begin_of_video|>",
|
|
"<|end_of_video|>",
|
|
"<|begin_of_audio|>",
|
|
"<|end_of_audio|>",
|
|
"<|begin_of_transcription|>",
|
|
"<|end_of_transcription|>"
|
|
],
|
|
"clean_up_tokenization_spaces": false,
|
|
"do_lower_case": false,
|
|
"eos_token": "<|endoftext|>",
|
|
"extra_special_tokens": {},
|
|
"model_max_length": 128000,
|
|
"pad_token": "<|endoftext|>",
|
|
"padding_side": "left",
|
|
"remove_space": false,
|
|
"tokenizer_class": "PreTrainedTokenizer"
|
|
}
|