Upload tokenizer

2024-11-14 06:17:45 +08:00
parent 97291a91ca
commit a5e6b5d98a
9 changed files with 2209 additions and 57 deletions
--- a/generation_config.json
+++ b/generation_config.json
@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 128000,
+  "eos_token_id": 128001,
+  "transformers_version": "4.44.2"
+}