commit dadac179e6c49046e3f9e74d2885a1fada05f6e0 Author: ModelHub XC Date: Wed Jun 17 16:17:17 2026 +0800 初始化项目，由ModelHub XC社区提供模型 Model: duyntnet/Mistral-Nemo-Instruct-2407-imatrix-GGUF Source: Original Platform diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..8940186 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,62 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +Mistral-Nemo-Instruct-2407-IQ1_S.gguf filter=lfs diff=lfs merge=lfs -text +Mistral-Nemo-Instruct-2407-IQ1_M.gguf filter=lfs diff=lfs merge=lfs -text +Mistral-Nemo-Instruct-2407-Q2_K.gguf filter=lfs diff=lfs merge=lfs -text +Mistral-Nemo-Instruct-2407-Q2_K_S.gguf filter=lfs diff=lfs merge=lfs -text +Mistral-Nemo-Instruct-2407-IQ2_S.gguf filter=lfs diff=lfs merge=lfs -text +Mistral-Nemo-Instruct-2407-IQ2_M.gguf filter=lfs diff=lfs merge=lfs -text +Mistral-Nemo-Instruct-2407-IQ2_XS.gguf filter=lfs diff=lfs merge=lfs -text +Mistral-Nemo-Instruct-2407-IQ2_XXS.gguf filter=lfs diff=lfs merge=lfs -text +Mistral-Nemo-Instruct-2407-Q3_K_S.gguf filter=lfs diff=lfs merge=lfs -text +Mistral-Nemo-Instruct-2407-Q3_K_M.gguf filter=lfs diff=lfs merge=lfs -text +Mistral-Nemo-Instruct-2407-Q3_K_L.gguf filter=lfs diff=lfs merge=lfs -text +Mistral-Nemo-Instruct-2407-IQ3_S.gguf filter=lfs diff=lfs merge=lfs -text +Mistral-Nemo-Instruct-2407-IQ3_M.gguf filter=lfs diff=lfs merge=lfs -text +Mistral-Nemo-Instruct-2407-IQ3_XS.gguf filter=lfs diff=lfs merge=lfs -text +Mistral-Nemo-Instruct-2407-IQ3_XXS.gguf filter=lfs diff=lfs merge=lfs -text +Mistral-Nemo-Instruct-2407-Q4_0.gguf filter=lfs diff=lfs merge=lfs -text +Mistral-Nemo-Instruct-2407-Q4_1.gguf filter=lfs diff=lfs merge=lfs -text +Mistral-Nemo-Instruct-2407-Q4_K_M.gguf filter=lfs diff=lfs merge=lfs -text +Mistral-Nemo-Instruct-2407-Q4_K_S.gguf filter=lfs diff=lfs merge=lfs -text +Mistral-Nemo-Instruct-2407-IQ4_NL.gguf filter=lfs diff=lfs merge=lfs -text +Mistral-Nemo-Instruct-2407-IQ4_XS.gguf filter=lfs diff=lfs merge=lfs -text +Mistral-Nemo-Instruct-2407-Q5_0.gguf filter=lfs diff=lfs merge=lfs -text +Mistral-Nemo-Instruct-2407-Q5_1.gguf filter=lfs diff=lfs merge=lfs -text +Mistral-Nemo-Instruct-2407-Q5_K_M.gguf filter=lfs diff=lfs merge=lfs -text +Mistral-Nemo-Instruct-2407-Q5_K_S.gguf filter=lfs diff=lfs merge=lfs -text +Mistral-Nemo-Instruct-2407-Q6_K.gguf filter=lfs diff=lfs merge=lfs -text +Mistral-Nemo-Instruct-2407-Q8_0.gguf filter=lfs diff=lfs merge=lfs -text diff --git a/Mistral-Nemo-Instruct-2407-IQ1_M.gguf b/Mistral-Nemo-Instruct-2407-IQ1_M.gguf new file mode 100644 index 0000000..fd75b6c --- /dev/null +++ b/Mistral-Nemo-Instruct-2407-IQ1_M.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dbebafc8a9938b65d66b917ecbb5c2b2313e6a2fe243b6bdfa11e1fd1808acc7 +size 3221625184 diff --git a/Mistral-Nemo-Instruct-2407-IQ1_S.gguf b/Mistral-Nemo-Instruct-2407-IQ1_S.gguf new file mode 100644 index 0000000..f7081c5 --- /dev/null +++ b/Mistral-Nemo-Instruct-2407-IQ1_S.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e1798defc43b0188add2bbc0d43d19dd3cdbec6c14d6c9c1c545fdf4928a4413 +size 2999212384 diff --git a/Mistral-Nemo-Instruct-2407-IQ2_M.gguf b/Mistral-Nemo-Instruct-2407-IQ2_M.gguf new file mode 100644 index 0000000..10b823d --- /dev/null +++ b/Mistral-Nemo-Instruct-2407-IQ2_M.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7d77d641be761cc1c9db8a8d0f76fd00f6b1f4d095ad6084fac24c78085b5998 +size 4435024224 diff --git a/Mistral-Nemo-Instruct-2407-IQ2_S.gguf b/Mistral-Nemo-Instruct-2407-IQ2_S.gguf new file mode 100644 index 0000000..847e740 --- /dev/null +++ b/Mistral-Nemo-Instruct-2407-IQ2_S.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ab2208cf6520b1ad4bae2b2ff07c5d671aefad13b141eb799a702e27cb2f39a7 +size 4138473824 diff --git a/Mistral-Nemo-Instruct-2407-IQ2_XS.gguf b/Mistral-Nemo-Instruct-2407-IQ2_XS.gguf new file mode 100644 index 0000000..208d80a --- /dev/null +++ b/Mistral-Nemo-Instruct-2407-IQ2_XS.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:016a6ac54964fdc93e518861c38b3303633c5558ee22bb575ed00d2176c38a64 +size 3915077984 diff --git a/Mistral-Nemo-Instruct-2407-IQ2_XXS.gguf b/Mistral-Nemo-Instruct-2407-IQ2_XXS.gguf new file mode 100644 index 0000000..48528a9 --- /dev/null +++ b/Mistral-Nemo-Instruct-2407-IQ2_XXS.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:61ab470a776083b9e999a795d9e215ff12875b1422eef8e2f04c6b5b66e84efd +size 3592313184 diff --git a/Mistral-Nemo-Instruct-2407-IQ3_M.gguf b/Mistral-Nemo-Instruct-2407-IQ3_M.gguf new file mode 100644 index 0000000..6c02289 --- /dev/null +++ b/Mistral-Nemo-Instruct-2407-IQ3_M.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:25d3c2aa841e71d20c59a6c6f512bda251d73f28e83e21761c25102bf5bb80b5 +size 5722233184 diff --git a/Mistral-Nemo-Instruct-2407-IQ3_S.gguf b/Mistral-Nemo-Instruct-2407-IQ3_S.gguf new file mode 100644 index 0000000..8d08568 --- /dev/null +++ b/Mistral-Nemo-Instruct-2407-IQ3_S.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0338fee950156b123ec148fe590a5eb89f876deb27f1fa0c6391427244513db0 +size 5562079584 diff --git a/Mistral-Nemo-Instruct-2407-IQ3_XS.gguf b/Mistral-Nemo-Instruct-2407-IQ3_XS.gguf new file mode 100644 index 0000000..0686998 --- /dev/null +++ b/Mistral-Nemo-Instruct-2407-IQ3_XS.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a83500275712097b0994d3e2d2556331efb672e562c5414c8af7ff3aaf27f1d4 +size 5306489184 diff --git a/Mistral-Nemo-Instruct-2407-IQ3_XXS.gguf b/Mistral-Nemo-Instruct-2407-IQ3_XXS.gguf new file mode 100644 index 0000000..cfb889c --- /dev/null +++ b/Mistral-Nemo-Instruct-2407-IQ3_XXS.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a275677290c9d3f6325f5d4753707a21701f1bc94851945b829d82924fb45274 +size 4945385824 diff --git a/Mistral-Nemo-Instruct-2407-IQ4_NL.gguf b/Mistral-Nemo-Instruct-2407-IQ4_NL.gguf new file mode 100644 index 0000000..06977c8 --- /dev/null +++ b/Mistral-Nemo-Instruct-2407-IQ4_NL.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2db15f56e3cc628b0665e45fd6e8a1c8adf1c1ceea8f28b23cdf3099aae631be +size 7097915744 diff --git a/Mistral-Nemo-Instruct-2407-IQ4_XS.gguf b/Mistral-Nemo-Instruct-2407-IQ4_XS.gguf new file mode 100644 index 0000000..54831d4 --- /dev/null +++ b/Mistral-Nemo-Instruct-2407-IQ4_XS.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d59e12be67e24620842998e26a7236794d2c9fa72f9853481d34fc776210930f +size 6742710624 diff --git a/Mistral-Nemo-Instruct-2407-Q2_K.gguf b/Mistral-Nemo-Instruct-2407-Q2_K.gguf new file mode 100644 index 0000000..6721e07 --- /dev/null +++ b/Mistral-Nemo-Instruct-2407-Q2_K.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:70844873ed499440da257cd4537ebf70a62e26dc3bfcf058be93023310af2fca +size 4791048544 diff --git a/Mistral-Nemo-Instruct-2407-Q2_K_S.gguf b/Mistral-Nemo-Instruct-2407-Q2_K_S.gguf new file mode 100644 index 0000000..c69ba7b --- /dev/null +++ b/Mistral-Nemo-Instruct-2407-Q2_K_S.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1e566bdd39a341dd8f0fca5f0eae04a5225572968fb444bb4a03f7197d2bc691 +size 4493678944 diff --git a/Mistral-Nemo-Instruct-2407-Q3_K_L.gguf b/Mistral-Nemo-Instruct-2407-Q3_K_L.gguf new file mode 100644 index 0000000..96f93b0 --- /dev/null +++ b/Mistral-Nemo-Instruct-2407-Q3_K_L.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d9e513f880e6c5d598e93154290e7d4017646dddf95d899a0c2c244ac8e26f22 +size 6561503584 diff --git a/Mistral-Nemo-Instruct-2407-Q3_K_M.gguf b/Mistral-Nemo-Instruct-2407-Q3_K_M.gguf new file mode 100644 index 0000000..6ad5b2e --- /dev/null +++ b/Mistral-Nemo-Instruct-2407-Q3_K_M.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ecce7c3eaf2b0334bf4b29cdfb5bbbc09b92953e75b13b184cb46aa6489624ef +size 6083090784 diff --git a/Mistral-Nemo-Instruct-2407-Q3_K_S.gguf b/Mistral-Nemo-Instruct-2407-Q3_K_S.gguf new file mode 100644 index 0000000..cbe3436 --- /dev/null +++ b/Mistral-Nemo-Instruct-2407-Q3_K_S.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cef895c37a7a01c797148b2ff2269da8b82b0d9d82030cdf3a9398739830a991 +size 5534226784 diff --git a/Mistral-Nemo-Instruct-2407-Q4_0.gguf b/Mistral-Nemo-Instruct-2407-Q4_0.gguf new file mode 100644 index 0000000..59dfd22 --- /dev/null +++ b/Mistral-Nemo-Instruct-2407-Q4_0.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5dbb82cf3b4fd7d8f38d811cbbcb448a3a52148c60976e5006447b51670aa222 +size 7094638944 diff --git a/Mistral-Nemo-Instruct-2407-Q4_1.gguf b/Mistral-Nemo-Instruct-2407-Q4_1.gguf new file mode 100644 index 0000000..a593a9a --- /dev/null +++ b/Mistral-Nemo-Instruct-2407-Q4_1.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bd94c5307b3ca02a734ef8cf323aa9e1964a81ad422be2bb25b332825f27aaea +size 7795218784 diff --git a/Mistral-Nemo-Instruct-2407-Q4_K_M.gguf b/Mistral-Nemo-Instruct-2407-Q4_K_M.gguf new file mode 100644 index 0000000..0444d58 --- /dev/null +++ b/Mistral-Nemo-Instruct-2407-Q4_K_M.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6e68a8fd463e0917de33321e817a9e1a0d2b5779db24841301efdc4406e580f2 +size 7477205344 diff --git a/Mistral-Nemo-Instruct-2407-Q4_K_S.gguf b/Mistral-Nemo-Instruct-2407-Q4_K_S.gguf new file mode 100644 index 0000000..66e30f7 --- /dev/null +++ b/Mistral-Nemo-Instruct-2407-Q4_K_S.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6b816d8b12e96d4b9a5f6a1f7f009ee7e835954d147a72f321151c480de694e3 +size 7120197984 diff --git a/Mistral-Nemo-Instruct-2407-Q5_0.gguf b/Mistral-Nemo-Instruct-2407-Q5_0.gguf new file mode 100644 index 0000000..42469e3 --- /dev/null +++ b/Mistral-Nemo-Instruct-2407-Q5_0.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3e3bd69402e3f02644b8bc6e9b969ddf45e07c22fae323bc652dbb99d9f9bff9 +size 8541673824 diff --git a/Mistral-Nemo-Instruct-2407-Q5_1.gguf b/Mistral-Nemo-Instruct-2407-Q5_1.gguf new file mode 100644 index 0000000..d67545f --- /dev/null +++ b/Mistral-Nemo-Instruct-2407-Q5_1.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b5cb487c3ff7a043dddf10e9f5ca71b45514f527539ee41a22739dacb4ef9436 +size 9242253664 diff --git a/Mistral-Nemo-Instruct-2407-Q5_K_M.gguf b/Mistral-Nemo-Instruct-2407-Q5_K_M.gguf new file mode 100644 index 0000000..6765da5 --- /dev/null +++ b/Mistral-Nemo-Instruct-2407-Q5_K_M.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:41d0712c80195dd58f1810a68915ab243a74c41393205c19f82abd18ddf8bd32 +size 8727632224 diff --git a/Mistral-Nemo-Instruct-2407-Q5_K_S.gguf b/Mistral-Nemo-Instruct-2407-Q5_K_S.gguf new file mode 100644 index 0000000..9c1457a --- /dev/null +++ b/Mistral-Nemo-Instruct-2407-Q5_K_S.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:858800c1f7fd112822c02e34cd271e74ce5dd1ab052e856c4e30786a3e38f4f3 +size 8518736224 diff --git a/Mistral-Nemo-Instruct-2407-Q6_K.gguf b/Mistral-Nemo-Instruct-2407-Q6_K.gguf new file mode 100644 index 0000000..f4dcee0 --- /dev/null +++ b/Mistral-Nemo-Instruct-2407-Q6_K.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e4e82039e134919e2e7cce55562a46e277fa5aa63d041e1bcc6d47d4f31a1001 +size 10056210784 diff --git a/Mistral-Nemo-Instruct-2407-Q8_0.gguf b/Mistral-Nemo-Instruct-2407-Q8_0.gguf new file mode 100644 index 0000000..f75e855 --- /dev/null +++ b/Mistral-Nemo-Instruct-2407-Q8_0.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:269a2828d16358a449ea0ccb39750cc9c4a2baa60762d39ae1c40fdfb4fced56 +size 13022370144 diff --git a/README.md b/README.md new file mode 100644 index 0000000..00b08c4 --- /dev/null +++ b/README.md @@ -0,0 +1,170 @@ +--- +license: other +language: +- en +pipeline_tag: text-generation +inference: false +tags: +- transformers +- gguf +- imatrix +- Mistral-Nemo-Instruct-2407 +--- +Quantizations of https://huggingface.co/mistralai/Mistral-Nemo-Instruct-2407 + + +### Inference Clients/UIs +* [llama.cpp](https://github.com/ggerganov/llama.cpp) +* [JanAI](https://github.com/janhq/jan) +* [KoboldCPP](https://github.com/LostRuins/koboldcpp) +* [text-generation-webui](https://github.com/oobabooga/text-generation-webui) +* [ollama](https://github.com/ollama/ollama) +* [GPT4All](https://github.com/nomic-ai/gpt4all) + +--- + +# From original readme + +## Usage + +The model can be used with three different frameworks + +- [`mistral_inference`](https://github.com/mistralai/mistral-inference): See [here](https://huggingface.co/mistralai/Mistral-Nemo-Instruct-2407#mistral-inference) +- [`transformers`](https://github.com/huggingface/transformers): See [here](#transformers) +- [`NeMo`](https://github.com/NVIDIA/NeMo): See [nvidia/Mistral-NeMo-12B-Instruct](https://huggingface.co/nvidia/Mistral-NeMo-12B-Instruct) + +### Mistral Inference + +#### Install + +It is recommended to use `mistralai/Mistral-Nemo-Instruct-2407` with [mistral-inference](https://github.com/mistralai/mistral-inference). For HF transformers code snippets, please keep scrolling. + +``` +pip install mistral_inference +``` + +#### Download + +```py +from huggingface_hub import snapshot_download +from pathlib import Path + +mistral_models_path = Path.home().joinpath('mistral_models', 'Nemo-Instruct') +mistral_models_path.mkdir(parents=True, exist_ok=True) + +snapshot_download(repo_id="mistralai/Mistral-Nemo-Instruct-2407", allow_patterns=["params.json", "consolidated.safetensors", "tekken.json"], local_dir=mistral_models_path) +``` + +#### Chat + +After installing `mistral_inference`, a `mistral-chat` CLI command should be available in your environment. You can chat with the model using + +``` +mistral-chat $HOME/mistral_models/Nemo-Instruct --instruct --max_tokens 256 --temperature 0.35 +``` + +*E.g.* Try out something like: +``` +How expensive would it be to ask a window cleaner to clean all windows in Paris. Make a reasonable guess in US Dollar. +``` + +#### Instruct following + +```py +from mistral_inference.transformer import Transformer +from mistral_inference.generate import generate + +from mistral_common.tokens.tokenizers.mistral import MistralTokenizer +from mistral_common.protocol.instruct.messages import UserMessage +from mistral_common.protocol.instruct.request import ChatCompletionRequest + +tokenizer = MistralTokenizer.from_file(f"{mistral_models_path}/tekken.json") +model = Transformer.from_folder(mistral_models_path) + +prompt = "How expensive would it be to ask a window cleaner to clean all windows in Paris. Make a reasonable guess in US Dollar." + +completion_request = ChatCompletionRequest(messages=[UserMessage(content=prompt)]) + +tokens = tokenizer.encode_chat_completion(completion_request).tokens + +out_tokens, _ = generate([tokens], model, max_tokens=64, temperature=0.35, eos_id=tokenizer.instruct_tokenizer.tokenizer.eos_id) +result = tokenizer.decode(out_tokens[0]) + +print(result) +``` + +#### Function calling + +```py +from mistral_common.protocol.instruct.tool_calls import Function, Tool +from mistral_inference.transformer import Transformer +from mistral_inference.generate import generate + +from mistral_common.tokens.tokenizers.mistral import MistralTokenizer +from mistral_common.protocol.instruct.messages import UserMessage +from mistral_common.protocol.instruct.request import ChatCompletionRequest + + +tokenizer = MistralTokenizer.from_file(f"{mistral_models_path}/tekken.json") +model = Transformer.from_folder(mistral_models_path) + +completion_request = ChatCompletionRequest( + tools=[ + Tool( + function=Function( + name="get_current_weather", + description="Get the current weather", + parameters={ + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", + }, + "format": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + "description": "The temperature unit to use. Infer this from the users location.", + }, + }, + "required": ["location", "format"], + }, + ) + ) + ], + messages=[ + UserMessage(content="What's the weather like today in Paris?"), + ], +) + +tokens = tokenizer.encode_chat_completion(completion_request).tokens + +out_tokens, _ = generate([tokens], model, max_tokens=256, temperature=0.35, eos_id=tokenizer.instruct_tokenizer.tokenizer.eos_id) +result = tokenizer.decode(out_tokens[0]) + +print(result) +``` + +### Transformers + +> [!IMPORTANT] +> NOTE: Until a new release has been made, you need to install transformers from source: +> ```sh +> pip install git+https://github.com/huggingface/transformers.git +> ``` + +If you want to use Hugging Face `transformers` to generate text, you can do something like this. + +```py +from transformers import pipeline + +messages = [ + {"role": "system", "content": "You are a pirate chatbot who always responds in pirate speak!"}, + {"role": "user", "content": "Who are you?"}, +] +chatbot = pipeline("text-generation", model="mistralai/Mistral-Nemo-Instruct-2407") +chatbot(messages) +``` + +> [!TIP] +> Unlike previous Mistral models, Mistral Nemo requires smaller temperatures. We recommend to use a temperature of 0.3. \ No newline at end of file