From ad112bb573db609057c2abe9199d3be7849c2eca Mon Sep 17 00:00:00 2001
From: x54-729 <whxiaohao@163.com>
Date: Mon, 26 Feb 2024 12:22:08 +0800
Subject: [PATCH] Add special tokens for chat

---
 special_tokens_map.json | 8 ++++++++
 tokenizer_config.json   | 8 ++++++++
 2 files changed, 16 insertions(+)

diff --git a/special_tokens_map.json b/special_tokens_map.json
index 9bfed75..925e61f 100644
--- a/special_tokens_map.json
+++ b/special_tokens_map.json
@@ -1,4 +1,12 @@
 {
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|action_start|>",
+    "<|action_end|>",
+    "<|interpreter|>",
+    "<|plugin|>"
+  ],
   "bos_token": "<s>",
   "eos_token": "</s>",
   "pad_token": "</s>",
diff --git a/tokenizer_config.json b/tokenizer_config.json
index 50ba041..d7e1a64 100644
--- a/tokenizer_config.json
+++ b/tokenizer_config.json
@@ -86,5 +86,13 @@
       "special": true
     }
   },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|action_start|>",
+    "<|action_end|>",
+    "<|interpreter|>",
+    "<|plugin|>"
+  ],
   "chat_template": "{{ bos_token }}{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}"
 }
\ No newline at end of file