Shanghai_AI_Laboratory/internlm2-math-20b
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

Add special tokens for chat

Browse Source
This commit is contained in:
x54-729
2024-02-26 12:22:08 +08:00
parent 2d5cd71608
commit ad112bb573
2 changed files with 16 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
special_tokens_map.json
View File

@@ -1,4 +1,12 @@
{ {
"additional_special_tokens": [
"<|im_start|>",
"<|im_end|>",
"<|action_start|>",
"<|action_end|>",
"<|interpreter|>",
"<|plugin|>"
],
"bos_token": "<s>", "bos_token": "<s>",
"eos_token": "</s>", "eos_token": "</s>",
"pad_token": "</s>", "pad_token": "</s>",

8
tokenizer_config.json
View File

@@ -86,5 +86,13 @@
"special": true "special": true
} }
}, },
"additional_special_tokens": [
"<|im_start|>",
"<|im_end|>",
"<|action_start|>",
"<|action_end|>",
"<|interpreter|>",
"<|plugin|>"
],
"chat_template": "{{ bos_token }}{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}" "chat_template": "{{ bos_token }}{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}"
} }