convert : add BPE pre-tokenization for DBRX (#7132)

* Add BPE pre-tokenization for DBRX. * Add vocab GGUFs. * Remove test. * Remove GGUFs.
2024-05-08 06:43:23 -04:00
parent 7e0b6a7b3b
commit 4cd621c26d
4 changed files with 9 additions and 0 deletions
--- a/llama.h
+++ b/llama.h
@@ -82,6 +82,7 @@ extern "C" {
        LLAMA_VOCAB_PRE_TYPE_REFACT         = 8,
        LLAMA_VOCAB_PRE_TYPE_COMMAND_R      = 9,
        LLAMA_VOCAB_PRE_TYPE_OLMO           = 10,
+        LLAMA_VOCAB_PRE_TYPE_DBRX           = 11,
    };

    // note: these values should be synchronized with ggml_rope