convert : fix gemma v1 tokenizer convert (#8248)

ggml-ci
2024-07-04 10:41:03 +03:00
parent f619024764
commit 20fc3804bf
28 changed files with 85 additions and 4 deletions
--- a/models/ggml-vocab-starcoder.gguf.inp
+++ b/models/ggml-vocab-starcoder.gguf.inp
@@ -91,6 +91,10 @@ __ggml_vocab_test__
 __ggml_vocab_test__
 333333333
 __ggml_vocab_test__
+Cửa Việt
+__ggml_vocab_test__
+ discards
+__ggml_vocab_test__