diff --git a/special_tokens_map.json b/special_tokens_map.json index 9bfed75..925e61f 100644 --- a/special_tokens_map.json +++ b/special_tokens_map.json @@ -1,4 +1,12 @@ { + "additional_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|action_start|>", + "<|action_end|>", + "<|interpreter|>", + "<|plugin|>" + ], "bos_token": "", "eos_token": "", "pad_token": "", diff --git a/tokenizer_config.json b/tokenizer_config.json index 50ba041..d7e1a64 100644 --- a/tokenizer_config.json +++ b/tokenizer_config.json @@ -86,5 +86,13 @@ "special": true } }, + "additional_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|action_start|>", + "<|action_end|>", + "<|interpreter|>", + "<|plugin|>" + ], "chat_template": "{{ bos_token }}{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}" } \ No newline at end of file