Add special tokens for chat

2024-02-26 12:22:08 +08:00
parent 2d5cd71608
commit ad112bb573
2 changed files with 16 additions and 0 deletions
--- a/special_tokens_map.json
+++ b/special_tokens_map.json
@@ -1,4 +1,12 @@
 {
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|action_start|>",
+    "<|action_end|>",
+    "<|interpreter|>",
+    "<|plugin|>"
+  ],
  "bos_token": "<s>",
  "eos_token": "</s>",
  "pad_token": "</s>",