Update for Transformers AWQ support

2025-02-15 00:56:32 +08:00
parent 9df68ab171
commit 4e56254ecb
15 changed files with 93932 additions and 55 deletions
--- a/special_tokens_map.json
+++ b/special_tokens_map.json
@@ -0,0 +1,6 @@
+{
+  "bos_token": "<s>",
+  "eos_token": "</s>",
+  "pad_token": "<unk>",
+  "unk_token": "<unk>"
+}