Upload tokenizer

2024-11-14 05:26:53 +08:00
parent 6ba437d324
commit 1c58f79c89
17 changed files with 94119 additions and 57 deletions
--- a/generation_config.json
+++ b/generation_config.json
@@ -0,0 +1,12 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": [
+    32000,
+    32001,
+    32007
+  ],
+  "max_length": 4096,
+  "pad_token_id": 32009,
+  "transformers_version": "4.44.2"
+}