llama : add Minerva 7B model support (#10673)

* Support for Minerva 7B * Update convert_hf_to_gguf_update.py
2024-12-05 19:30:59 +01:00
parent 0cd182ebcc
commit 6fe6247831
5 changed files with 9 additions and 0 deletions
--- a/src/llama.cpp
+++ b/src/llama.cpp
@@ -6479,6 +6479,9 @@ static void llm_load_vocab(
                vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_CHAMELEON;
                vocab.tokenizer_add_bos = true;
                vocab.tokenizer_clean_spaces = false;
+            } else if (
+                tokenizer_pre == "minerva-7b") {
+                vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_MINERVA;
            } else {
                throw std::runtime_error(format("unknown pre-tokenizer type: '%s'", tokenizer_pre.c_str()));
            }