Upload tokenizer

2024-11-14 05:26:53 +08:00
parent 6ba437d324
commit 1c58f79c89
17 changed files with 94119 additions and 57 deletions
--- a/added_tokens.json
+++ b/added_tokens.json
@@ -0,0 +1,13 @@
+{
+  "<|assistant|>": 32001,
+  "<|endoftext|>": 32000,
+  "<|end|>": 32007,
+  "<|placeholder1|>": 32002,
+  "<|placeholder2|>": 32003,
+  "<|placeholder3|>": 32004,
+  "<|placeholder4|>": 32005,
+  "<|placeholder5|>": 32008,
+  "<|placeholder6|>": 32009,
+  "<|system|>": 32006,
+  "<|user|>": 32010
+}