Upload tokenizer

2025-01-15 13:07:29 +08:00
parent 0e9bcf94a7
commit 16a3b21282
13 changed files with 23549 additions and 65 deletions
--- a/generation_config.json
+++ b/generation_config.json
@@ -0,0 +1,10 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 128000,
+  "eos_token_id": [
+    128001,
+    128008,
+    128009
+  ],
+  "transformers_version": "4.48.0"
+}