147 lines
4.3 KiB
JSON
147 lines
4.3 KiB
JSON
|
{
|
|||
|
|
"added_tokens_decoder": {
|
|||
|
|
"151329": {
|
|||
|
|
"content": "<|endoftext|>",
|
|||
|
|
"lstrip": false,
|
|||
|
|
"normalized": false,
|
|||
|
|
"rstrip": false,
|
|||
|
|
"single_word": false,
|
|||
|
|
"special": true
|
|||
|
|
},
|
|||
|
|
"151330": {
|
|||
|
|
"content": "[MASK]",
|
|||
|
|
"lstrip": false,
|
|||
|
|
"normalized": false,
|
|||
|
|
"rstrip": false,
|
|||
|
|
"single_word": false,
|
|||
|
|
"special": true
|
|||
|
|
},
|
|||
|
|
"151331": {
|
|||
|
|
"content": "[gMASK]",
|
|||
|
|
"lstrip": false,
|
|||
|
|
"normalized": false,
|
|||
|
|
"rstrip": false,
|
|||
|
|
"single_word": false,
|
|||
|
|
"special": true
|
|||
|
|
},
|
|||
|
|
"151332": {
|
|||
|
|
"content": "[sMASK]",
|
|||
|
|
"lstrip": false,
|
|||
|
|
"normalized": false,
|
|||
|
|
"rstrip": false,
|
|||
|
|
"single_word": false,
|
|||
|
|
"special": true
|
|||
|
|
},
|
|||
|
|
"151333": {
|
|||
|
|
"content": "<sop>",
|
|||
|
|
"lstrip": false,
|
|||
|
|
"normalized": false,
|
|||
|
|
"rstrip": false,
|
|||
|
|
"single_word": false,
|
|||
|
|
"special": true
|
|||
|
|
},
|
|||
|
|
"151334": {
|
|||
|
|
"content": "<eop>",
|
|||
|
|
"lstrip": false,
|
|||
|
|
"normalized": false,
|
|||
|
|
"rstrip": false,
|
|||
|
|
"single_word": false,
|
|||
|
|
"special": true
|
|||
|
|
},
|
|||
|
|
"151335": {
|
|||
|
|
"content": "<|system|>",
|
|||
|
|
"lstrip": false,
|
|||
|
|
"normalized": false,
|
|||
|
|
"rstrip": false,
|
|||
|
|
"single_word": false,
|
|||
|
|
"special": true
|
|||
|
|
},
|
|||
|
|
"151336": {
|
|||
|
|
"content": "<|user|>",
|
|||
|
|
"lstrip": false,
|
|||
|
|
"normalized": false,
|
|||
|
|
"rstrip": false,
|
|||
|
|
"single_word": false,
|
|||
|
|
"special": true
|
|||
|
|
},
|
|||
|
|
"151337": {
|
|||
|
|
"content": "<|assistant|>",
|
|||
|
|
"lstrip": false,
|
|||
|
|
"normalized": false,
|
|||
|
|
"rstrip": false,
|
|||
|
|
"single_word": false,
|
|||
|
|
"special": true
|
|||
|
|
},
|
|||
|
|
"151338": {
|
|||
|
|
"content": "<|observation|>",
|
|||
|
|
"lstrip": false,
|
|||
|
|
"normalized": false,
|
|||
|
|
"rstrip": false,
|
|||
|
|
"single_word": false,
|
|||
|
|
"special": true
|
|||
|
|
},
|
|||
|
|
"151339": {
|
|||
|
|
"content": "<|begin_of_image|>",
|
|||
|
|
"lstrip": false,
|
|||
|
|
"normalized": false,
|
|||
|
|
"rstrip": false,
|
|||
|
|
"single_word": false,
|
|||
|
|
"special": true
|
|||
|
|
},
|
|||
|
|
"151340": {
|
|||
|
|
"content": "<|end_of_image|>",
|
|||
|
|
"lstrip": false,
|
|||
|
|
"normalized": false,
|
|||
|
|
"rstrip": false,
|
|||
|
|
"single_word": false,
|
|||
|
|
"special": true
|
|||
|
|
},
|
|||
|
|
"151341": {
|
|||
|
|
"content": "<|begin_of_video|>",
|
|||
|
|
"lstrip": false,
|
|||
|
|
"normalized": false,
|
|||
|
|
"rstrip": false,
|
|||
|
|
"single_word": false,
|
|||
|
|
"special": true
|
|||
|
|
},
|
|||
|
|
"151342": {
|
|||
|
|
"content": "<|end_of_video|>",
|
|||
|
|
"lstrip": false,
|
|||
|
|
"normalized": false,
|
|||
|
|
"rstrip": false,
|
|||
|
|
"single_word": false,
|
|||
|
|
"special": true
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"additional_special_tokens": [
|
|||
|
|
"<|endoftext|>",
|
|||
|
|
"[MASK]",
|
|||
|
|
"[gMASK]",
|
|||
|
|
"[sMASK]",
|
|||
|
|
"<sop>",
|
|||
|
|
"<eop>",
|
|||
|
|
"<|system|>",
|
|||
|
|
"<|user|>",
|
|||
|
|
"<|assistant|>",
|
|||
|
|
"<|observation|>",
|
|||
|
|
"<|begin_of_image|>",
|
|||
|
|
"<|end_of_image|>",
|
|||
|
|
"<|begin_of_video|>",
|
|||
|
|
"<|end_of_video|>"
|
|||
|
|
],
|
|||
|
|
"chat_template": "[gMASK]<sop>{%- if tools -%}<|system|>你是一个名为 ChatGLM 的人工智能助手。你是基于智谱 AI 公司训练的语言模型 GLM-4 模型开发的，你的任务是针对用户的问题和要求提供适当的答复和支持。\n\n# 可用工具\n\n{% for tool in tools %}{%- set function = tool.function if tool.get(\"function\") else tool %}\n\n## {{ function.name }}\n\n{{ function | tojson(indent=4, ensure_ascii=False) }}\n在调用上述函数时，请使用 Json 格式表示调用的参数。{%- endfor %}{%- endif -%}{%- for msg in messages %}{%- if msg.role == 'system' %}<|system|>\n{{ msg.content }}{%- endif %}{%- endfor %}{%- for message in messages if message.role != 'system' %}{%- set role = message['role'] %}{%- set content = message['content'] %}{%- set visible = content.split('</think>')[-1].strip() %}{%- set meta = message.get(\"metadata\", \"\") %}{%- if role == 'user' %}<|user|>\n{{ visible }}{%- elif role == 'assistant' and not meta %}<|assistant|>\n{{ visible }}{%- elif role == 'assistant' and meta %}<|assistant|>{{ meta }} \n{{ visible }}{%- elif role == 'observation' %}<|observation|>\n{{ visible }}{%- endif %}{%- endfor %}{% if add_generation_prompt %}<|assistant|>{% endif %}",
|
|||
|
|
"clean_up_tokenization_spaces": false,
|
|||
|
|
"do_lower_case": false,
|
|||
|
|
"eos_token": "<|endoftext|>",
|
|||
|
|
"extra_special_tokens": {},
|
|||
|
|
"model_input_names": [
|
|||
|
|
"input_ids",
|
|||
|
|
"attention_mask"
|
|||
|
|
],
|
|||
|
|
"model_max_length": 128000,
|
|||
|
|
"pad_token": "<|endoftext|>",
|
|||
|
|
"padding_side": "left",
|
|||
|
|
"remove_space": false,
|
|||
|
|
"tokenizer_class": "PreTrainedTokenizer"
|
|||
|
|
}
|