234 lines
7.6 KiB
JSON
234 lines
7.6 KiB
JSON
{
|
|
"add_prefix_space": false,
|
|
"backend": "tokenizers",
|
|
"bos_token": null,
|
|
"clean_up_tokenization_spaces": false,
|
|
"eos_token": "<|im_end|>",
|
|
"errors": "replace",
|
|
"is_local": false,
|
|
"model_max_length": 262144,
|
|
"pad_token": "<|PAD_TOKEN|>",
|
|
"padding_side": "left",
|
|
"split_special_tokens": false,
|
|
"tokenizer_class": "Qwen2Tokenizer",
|
|
"unk_token": null,
|
|
"added_tokens_decoder": {
|
|
"151643": {
|
|
"content": "<|endoftext|>",
|
|
"single_word": false,
|
|
"lstrip": false,
|
|
"rstrip": false,
|
|
"normalized": false,
|
|
"special": true
|
|
},
|
|
"151644": {
|
|
"content": "<|im_start|>",
|
|
"single_word": false,
|
|
"lstrip": false,
|
|
"rstrip": false,
|
|
"normalized": false,
|
|
"special": true
|
|
},
|
|
"151645": {
|
|
"content": "<|im_end|>",
|
|
"single_word": false,
|
|
"lstrip": false,
|
|
"rstrip": false,
|
|
"normalized": false,
|
|
"special": true
|
|
},
|
|
"151646": {
|
|
"content": "<|object_ref_start|>",
|
|
"single_word": false,
|
|
"lstrip": false,
|
|
"rstrip": false,
|
|
"normalized": false,
|
|
"special": true
|
|
},
|
|
"151647": {
|
|
"content": "<|object_ref_end|>",
|
|
"single_word": false,
|
|
"lstrip": false,
|
|
"rstrip": false,
|
|
"normalized": false,
|
|
"special": true
|
|
},
|
|
"151648": {
|
|
"content": "<|box_start|>",
|
|
"single_word": false,
|
|
"lstrip": false,
|
|
"rstrip": false,
|
|
"normalized": false,
|
|
"special": true
|
|
},
|
|
"151649": {
|
|
"content": "<|box_end|>",
|
|
"single_word": false,
|
|
"lstrip": false,
|
|
"rstrip": false,
|
|
"normalized": false,
|
|
"special": true
|
|
},
|
|
"151650": {
|
|
"content": "<|quad_start|>",
|
|
"single_word": false,
|
|
"lstrip": false,
|
|
"rstrip": false,
|
|
"normalized": false,
|
|
"special": true
|
|
},
|
|
"151651": {
|
|
"content": "<|quad_end|>",
|
|
"single_word": false,
|
|
"lstrip": false,
|
|
"rstrip": false,
|
|
"normalized": false,
|
|
"special": true
|
|
},
|
|
"151652": {
|
|
"content": "<|vision_start|>",
|
|
"single_word": false,
|
|
"lstrip": false,
|
|
"rstrip": false,
|
|
"normalized": false,
|
|
"special": true
|
|
},
|
|
"151653": {
|
|
"content": "<|vision_end|>",
|
|
"single_word": false,
|
|
"lstrip": false,
|
|
"rstrip": false,
|
|
"normalized": false,
|
|
"special": true
|
|
},
|
|
"151654": {
|
|
"content": "<|vision_pad|>",
|
|
"single_word": false,
|
|
"lstrip": false,
|
|
"rstrip": false,
|
|
"normalized": false,
|
|
"special": true
|
|
},
|
|
"151655": {
|
|
"content": "<|image_pad|>",
|
|
"single_word": false,
|
|
"lstrip": false,
|
|
"rstrip": false,
|
|
"normalized": false,
|
|
"special": true
|
|
},
|
|
"151656": {
|
|
"content": "<|video_pad|>",
|
|
"single_word": false,
|
|
"lstrip": false,
|
|
"rstrip": false,
|
|
"normalized": false,
|
|
"special": true
|
|
},
|
|
"151657": {
|
|
"content": "<tool_call>",
|
|
"single_word": false,
|
|
"lstrip": false,
|
|
"rstrip": false,
|
|
"normalized": false,
|
|
"special": false
|
|
},
|
|
"151658": {
|
|
"content": "</tool_call>",
|
|
"single_word": false,
|
|
"lstrip": false,
|
|
"rstrip": false,
|
|
"normalized": false,
|
|
"special": false
|
|
},
|
|
"151659": {
|
|
"content": "<|fim_prefix|>",
|
|
"single_word": false,
|
|
"lstrip": false,
|
|
"rstrip": false,
|
|
"normalized": false,
|
|
"special": false
|
|
},
|
|
"151660": {
|
|
"content": "<|fim_middle|>",
|
|
"single_word": false,
|
|
"lstrip": false,
|
|
"rstrip": false,
|
|
"normalized": false,
|
|
"special": false
|
|
},
|
|
"151661": {
|
|
"content": "<|fim_suffix|>",
|
|
"single_word": false,
|
|
"lstrip": false,
|
|
"rstrip": false,
|
|
"normalized": false,
|
|
"special": false
|
|
},
|
|
"151662": {
|
|
"content": "<|fim_pad|>",
|
|
"single_word": false,
|
|
"lstrip": false,
|
|
"rstrip": false,
|
|
"normalized": false,
|
|
"special": false
|
|
},
|
|
"151663": {
|
|
"content": "<|repo_name|>",
|
|
"single_word": false,
|
|
"lstrip": false,
|
|
"rstrip": false,
|
|
"normalized": false,
|
|
"special": false
|
|
},
|
|
"151664": {
|
|
"content": "<|file_sep|>",
|
|
"single_word": false,
|
|
"lstrip": false,
|
|
"rstrip": false,
|
|
"normalized": false,
|
|
"special": false
|
|
},
|
|
"151665": {
|
|
"content": "<tool_response>",
|
|
"single_word": false,
|
|
"lstrip": false,
|
|
"rstrip": false,
|
|
"normalized": false,
|
|
"special": false
|
|
},
|
|
"151666": {
|
|
"content": "</tool_response>",
|
|
"single_word": false,
|
|
"lstrip": false,
|
|
"rstrip": false,
|
|
"normalized": false,
|
|
"special": false
|
|
},
|
|
"151667": {
|
|
"content": "<think>",
|
|
"single_word": false,
|
|
"lstrip": false,
|
|
"rstrip": false,
|
|
"normalized": false,
|
|
"special": false
|
|
},
|
|
"151668": {
|
|
"content": "</think>",
|
|
"single_word": false,
|
|
"lstrip": false,
|
|
"rstrip": false,
|
|
"normalized": false,
|
|
"special": false
|
|
},
|
|
"151669": {
|
|
"content": "<|PAD_TOKEN|>",
|
|
"single_word": false,
|
|
"lstrip": false,
|
|
"rstrip": false,
|
|
"normalized": false,
|
|
"special": true
|
|
}
|
|
},
|
|
"chat_template": "{%- macro strip_thinking(text) -%}\n {%- set ns = namespace(result='') -%}\n {%- for part in text.split('<channel|>') -%}\n {%- if '<|channel>' in part -%}\n {%- set ns.result = ns.result + part.split('<|channel>')[0] -%}\n {%- else -%}\n {%- set ns.result = ns.result + part -%}\n {%- endif -%}\n {%- endfor -%}\n {{- ns.result | trim -}}\n{%- endmacro -%}\n{%- set thinking = enable_thinking is defined and enable_thinking -%}\n{%- set loop_messages = messages -%}\n{%- if messages[0]['role'] in ['system', 'developer'] or thinking -%}\n {{ '<|turn>system\n' }}\n {%- if thinking -%}\n {{ '<|think|>\n' }}\n {%- endif -%}\n {%- if messages[0]['role'] in ['system', 'developer'] -%}\n {{ messages[0]['content'] | trim }}\n {%- set loop_messages = messages[1:] -%}\n {%- endif -%}\n {{ '<turn|>\n' }}\n{%- endif -%}\n{%- for message in loop_messages -%}\n {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%}\n {{ raise_exception(\"Conversation roles must alternate user/assistant/user/assistant/...\") }}\n {%- endif -%}\n {%- if (message['role'] == 'assistant') -%}\n {%- set role = \"model\" -%}\n {%- else -%}\n {%- set role = message['role'] -%}\n {%- endif -%}\n {{ '<|turn>' + role + '\n' }}\n {%- if message['content'] is string -%}\n {%- if role == \"model\" -%}\n {{ strip_thinking(message['content']) }}\n {%- else -%}\n {{ message['content'] | trim }}\n {%- endif -%}\n {%- elif message['content'] is iterable -%}\n {%- for item in message['content'] -%}\n {%- if item['type'] == 'audio' -%}\n {{ '<|audio|>' }}\n {%- elif item['type'] == 'image' -%}\n {{ '<|image|>' }}\n {%- elif item['type'] == 'video' -%}\n {{ '<|video|>' }}\n {%- elif item['type'] == 'text' -%}\n {%- if role == \"model\" -%}\n {{ strip_thinking(item['text']) }}\n {%- else -%}\n {{ item['text'] | trim }}\n {%- endif -%}\n {%- endif -%}\n {%- endfor -%}\n {%- else -%}\n {{ raise_exception(\"Invalid content type\") }}\n {%- endif -%}\n {{ '<turn|>\n' }}\n{%- endfor -%}\n{%- if add_generation_prompt -%}\n {{'<|turn>model\n'}}\n{%- endif -%}\n"
|
|
} |