Upload tokenizer

2025-01-05 14:34:00 +08:00
parent 13241bfbdd
commit 96a47304af
12 changed files with 156550 additions and 65 deletions
--- a/special_tokens_map.json
+++ b/special_tokens_map.json
@@ -0,0 +1,39 @@
+{
+  "additional_special_tokens": [
+    {
+      "content": "<|sound_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|sound_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|text_to_sementic|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    }
+  ],
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}