commit 77abbce1ec17f672d394eb2fbba51d67cf1dab12 Author: ModelHub XC Date: Tue Sep 8 13:38:16 2026 +0800 初始化项目，由ModelHub XC社区提供模型 Model: CISCai/Mistral-7B-Instruct-v0.3-SOTA-GGUF Source: Original Platform diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..15112cd --- /dev/null +++ b/.gitattributes @@ -0,0 +1,48 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +Mistral-7B-Instruct-v0.3.fp16.gguf filter=lfs diff=lfs merge=lfs -text +Mistral-7B-Instruct-v0.3.imatrix.dat filter=lfs diff=lfs merge=lfs -text +Mistral-7B-Instruct-v0.3.IQ1_M.gguf filter=lfs diff=lfs merge=lfs -text +Mistral-7B-Instruct-v0.3.IQ1_S.gguf filter=lfs diff=lfs merge=lfs -text +Mistral-7B-Instruct-v0.3.IQ2_M.gguf filter=lfs diff=lfs merge=lfs -text +Mistral-7B-Instruct-v0.3.IQ2_S.gguf filter=lfs diff=lfs merge=lfs -text +Mistral-7B-Instruct-v0.3.IQ2_XS.gguf filter=lfs diff=lfs merge=lfs -text +Mistral-7B-Instruct-v0.3.IQ2_XXS.gguf filter=lfs diff=lfs merge=lfs -text +Mistral-7B-Instruct-v0.3.IQ3_M.gguf filter=lfs diff=lfs merge=lfs -text +Mistral-7B-Instruct-v0.3.IQ3_S.gguf filter=lfs diff=lfs merge=lfs -text +Mistral-7B-Instruct-v0.3.IQ3_XS.gguf filter=lfs diff=lfs merge=lfs -text +Mistral-7B-Instruct-v0.3.IQ3_XXS.gguf filter=lfs diff=lfs merge=lfs -text +Mistral-7B-Instruct-v0.3.IQ4_XS.gguf filter=lfs diff=lfs merge=lfs -text diff --git a/Mistral-7B-Instruct-v0.3.IQ1_M.gguf b/Mistral-7B-Instruct-v0.3.IQ1_M.gguf new file mode 100644 index 0000000..11b336c --- /dev/null +++ b/Mistral-7B-Instruct-v0.3.IQ1_M.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6d1a2dd797732b2bd81522c0dc2ce07f02c9a760f760e16302dda6477d1b7b59 +size 1757664288 diff --git a/Mistral-7B-Instruct-v0.3.IQ1_S.gguf b/Mistral-7B-Instruct-v0.3.IQ1_S.gguf new file mode 100644 index 0000000..e30fadc --- /dev/null +++ b/Mistral-7B-Instruct-v0.3.IQ1_S.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:23a3cfd34d22263ccf49dfc02e7f3ea7c983118680a9dedca31817efdbd3efab +size 1615320096 diff --git a/Mistral-7B-Instruct-v0.3.IQ2_M.gguf b/Mistral-7B-Instruct-v0.3.IQ2_M.gguf new file mode 100644 index 0000000..7853c79 --- /dev/null +++ b/Mistral-7B-Instruct-v0.3.IQ2_M.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7e93031a8c059e014a904b2f8cebdf073ff7310f9a6d2579c07c8d6ebdc6d8ae +size 2504250400 diff --git a/Mistral-7B-Instruct-v0.3.IQ2_S.gguf b/Mistral-7B-Instruct-v0.3.IQ2_S.gguf new file mode 100644 index 0000000..0ceb4b7 --- /dev/null +++ b/Mistral-7B-Instruct-v0.3.IQ2_S.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2afeb6b4536c43014c5575aed66ce0577c965b9bf5019bc7c43cdbbaf520bc4a +size 2314458144 diff --git a/Mistral-7B-Instruct-v0.3.IQ2_XS.gguf b/Mistral-7B-Instruct-v0.3.IQ2_XS.gguf new file mode 100644 index 0000000..a463cfa --- /dev/null +++ b/Mistral-7B-Instruct-v0.3.IQ2_XS.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4ff8d8d252da2955e69420ddf6ba79c7f9364c52a269d3333edefb6ca4e58bac +size 2201474080 diff --git a/Mistral-7B-Instruct-v0.3.IQ2_XXS.gguf b/Mistral-7B-Instruct-v0.3.IQ2_XXS.gguf new file mode 100644 index 0000000..72e4f2f --- /dev/null +++ b/Mistral-7B-Instruct-v0.3.IQ2_XXS.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9ebea120c9f96af5595fbb1b7addc8daa7841bfcf16703d0382d21b22e66a8e5 +size 1994904608 diff --git a/Mistral-7B-Instruct-v0.3.IQ3_M.gguf b/Mistral-7B-Instruct-v0.3.IQ3_M.gguf new file mode 100644 index 0000000..eb3866e --- /dev/null +++ b/Mistral-7B-Instruct-v0.3.IQ3_M.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8d49160a4dbf6af69ab8a22bcee90d821e724aa0deddd63c2625f60eaeff4af4 +size 3288847392 diff --git a/Mistral-7B-Instruct-v0.3.IQ3_S.gguf b/Mistral-7B-Instruct-v0.3.IQ3_S.gguf new file mode 100644 index 0000000..1b82521 --- /dev/null +++ b/Mistral-7B-Instruct-v0.3.IQ3_S.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:156cbd42ee5a635bb795dc36bd3283d7c746b2e12bfcd451163756a283345e0c +size 3186349088 diff --git a/Mistral-7B-Instruct-v0.3.IQ3_XS.gguf b/Mistral-7B-Instruct-v0.3.IQ3_XS.gguf new file mode 100644 index 0000000..811e8e5 --- /dev/null +++ b/Mistral-7B-Instruct-v0.3.IQ3_XS.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:84985ba223dc145565068e1d1b617cc44201c46ad87b009c7281e708613bf484 +size 3022771232 diff --git a/Mistral-7B-Instruct-v0.3.IQ3_XXS.gguf b/Mistral-7B-Instruct-v0.3.IQ3_XXS.gguf new file mode 100644 index 0000000..f59f85a --- /dev/null +++ b/Mistral-7B-Instruct-v0.3.IQ3_XXS.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:26233eb5e3fc54bbdfe977934ef7b015dd5dac759e424710f024fc795781027c +size 2830881824 diff --git a/Mistral-7B-Instruct-v0.3.IQ4_XS.gguf b/Mistral-7B-Instruct-v0.3.IQ4_XS.gguf new file mode 100644 index 0000000..f57e006 --- /dev/null +++ b/Mistral-7B-Instruct-v0.3.IQ4_XS.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0afba232e1e46573db08bd965c82ef583f0b71131be840a19982339d94e067c +size 3911963680 diff --git a/Mistral-7B-Instruct-v0.3.fp16.gguf b/Mistral-7B-Instruct-v0.3.fp16.gguf new file mode 100644 index 0000000..c81902f --- /dev/null +++ b/Mistral-7B-Instruct-v0.3.fp16.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dbd62e09c7201d790605d3a8be94260bfbecd70c470ed297e4d584fa319ed8ab +size 14497338144 diff --git a/Mistral-7B-Instruct-v0.3.imatrix.dat b/Mistral-7B-Instruct-v0.3.imatrix.dat new file mode 100644 index 0000000..4a6aced --- /dev/null +++ b/Mistral-7B-Instruct-v0.3.imatrix.dat @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e44d933bda5232dd9081c20e0ae2ef2df5aab5c4388fc308b237d1454e694e81 +size 4988162 diff --git a/README.md b/README.md new file mode 100644 index 0000000..d5ec931 --- /dev/null +++ b/README.md @@ -0,0 +1,290 @@ +--- +base_model: mistralai/Mistral-7B-Instruct-v0.3 +language: +- en +pipeline_tag: text-generation +license: apache-2.0 +model_creator: Mistral AI +model_name: Mistral-7B-Instruct-v0.3 +model_type: mistral +quantized_by: CISC +--- + +# Mistral-7B-Instruct-v0.3 - SOTA GGUF +- Model creator: [Mistral AI](https://huggingface.co/mistralai) +- Original model: [Mistral-7B-Instruct-v0.3](https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.3) + + +## Description + +This repo contains State Of The Art quantized GGUF format model files for [Mistral-7B-Instruct-v0.3](https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.3). + +Quantization was done with an importance matrix that was trained for ~1M tokens (256 batches of 4096 tokens) of [groups_merged.txt](https://github.com/ggerganov/llama.cpp/discussions/5263#discussioncomment-8395384) and [wiki.train.raw](https://raw.githubusercontent.com/pytorch/examples/main/word_language_model/data/wikitext-2/train.txt) concatenated. + +The embedded chat template has been extended to support function calling via OpenAI-compatible `tools` parameter, see [example](#simple-llama-cpp-python-example-function-calling-code). + + + + + +## Prompt template: Mistral v3 + +``` +[AVAILABLE_TOOLS] [{"name": "function_name", "description": "Description", "parameters": {...}}, ...][/AVAILABLE_TOOLS][INST] {prompt}[/INST] +``` + + + + + +## Compatibility + +These quantised GGUFv3 files are compatible with llama.cpp from February 27th 2024 onwards, as of commit [0becb22](https://github.com/ggerganov/llama.cpp/commit/0becb22ac05b6542bd9d5f2235691aa1d3d4d307) + +They are also compatible with many third party UIs and libraries provided they are built using a recent llama.cpp. + +## Explanation of quantisation methods + +
+ Click to see details + +The new methods available are: + +* GGML_TYPE_IQ1_S - 1-bit quantization in super-blocks with an importance matrix applied, effectively using 1.56 bits per weight (bpw) +* GGML_TYPE_IQ1_M - 1-bit quantization in super-blocks with an importance matrix applied, effectively using 1.75 bpw +* GGML_TYPE_IQ2_XXS - 2-bit quantization in super-blocks with an importance matrix applied, effectively using 2.06 bpw +* GGML_TYPE_IQ2_XS - 2-bit quantization in super-blocks with an importance matrix applied, effectively using 2.31 bpw +* GGML_TYPE_IQ2_S - 2-bit quantization in super-blocks with an importance matrix applied, effectively using 2.5 bpw +* GGML_TYPE_IQ2_M - 2-bit quantization in super-blocks with an importance matrix applied, effectively using 2.7 bpw +* GGML_TYPE_IQ3_XXS - 3-bit quantization in super-blocks with an importance matrix applied, effectively using 3.06 bpw +* GGML_TYPE_IQ3_XS - 3-bit quantization in super-blocks with an importance matrix applied, effectively using 3.3 bpw +* GGML_TYPE_IQ3_S - 3-bit quantization in super-blocks with an importance matrix applied, effectively using 3.44 bpw +* GGML_TYPE_IQ3_M - 3-bit quantization in super-blocks with an importance matrix applied, effectively using 3.66 bpw +* GGML_TYPE_IQ4_XS - 4-bit quantization in super-blocks with an importance matrix applied, effectively using 4.25 bpw +* GGML_TYPE_IQ4_NL - 4-bit non-linearly mapped quantization with an importance matrix applied, effectively using 4.5 bpw + +Refer to the Provided Files table below to see what files use which methods, and how. +
+ + + +## Provided files + +| Name | Quant method | Bits | Size | Max RAM required | Use case | +| ---- | ---- | ---- | ---- | ---- | ----- | +| [Mistral-7B-Instruct-v0.3.IQ1_S.gguf](https://huggingface.co/CISCai/Mistral-7B-Instruct-v0.3-SOTA-GGUF/blob/main/Mistral-7B-Instruct-v0.3.IQ1_S.gguf) | IQ1_S | 1 | 1.5 GB| 2.5 GB | smallest, significant quality loss - **TBD**: Waiting for [this issue](https://github.com/ggerganov/llama.cpp/issues/5996) to be resolved | +| [Mistral-7B-Instruct-v0.3.IQ1_M.gguf](https://huggingface.co/CISCai/Mistral-7B-Instruct-v0.3-SOTA-GGUF/blob/main/Mistral-7B-Instruct-v0.3.IQ1_M.gguf) | IQ1_M | 1 | 1.6 GB| 2.6 GB | very small, significant quality loss | +| [Mistral-7B-Instruct-v0.3.IQ2_XXS.gguf](https://huggingface.co/CISCai/Mistral-7B-Instruct-v0.3-SOTA-GGUF/blob/main/Mistral-7B-Instruct-v0.3.IQ2_XXS.gguf) | IQ2_XXS | 2 | 1.8 GB| 2.8 GB | very small, high quality loss | +| [Mistral-7B-Instruct-v0.3.IQ2_XS.gguf](https://huggingface.co/CISCai/Mistral-7B-Instruct-v0.3-SOTA-GGUF/blob/main/Mistral-7B-Instruct-v0.3.IQ2_XS.gguf) | IQ2_XS | 2 | 1.9 GB| 2.9 GB | very small, high quality loss | +| [Mistral-7B-Instruct-v0.3.IQ2_S.gguf](https://huggingface.co/CISCai/Mistral-7B-Instruct-v0.3-SOTA-GGUF/blob/main/Mistral-7B-Instruct-v0.3.IQ2_S.gguf) | IQ2_S | 2 | 2.1 GB| 3.1 GB | small, substantial quality loss | +| [Mistral-7B-Instruct-v0.3.IQ2_M.gguf](https://huggingface.co/CISCai/Mistral-7B-Instruct-v0.3-SOTA-GGUF/blob/main/Mistral-7B-Instruct-v0.3.IQ2_M.gguf) | IQ2_M | 2 | 2.2 GB| 3.2 GB | small, greater quality loss | +| [Mistral-7B-Instruct-v0.3.IQ3_XXS.gguf](https://huggingface.co/CISCai/Mistral-7B-Instruct-v0.3-SOTA-GGUF/blob/main/Mistral-7B-Instruct-v0.3.IQ3_XXS.gguf) | IQ3_XXS | 3 | 2.5 GB| 3.5 GB | very small, high quality loss | +| [Mistral-7B-Instruct-v0.3.IQ3_XS.gguf](https://huggingface.co/CISCai/Mistral-7B-Instruct-v0.3-SOTA-GGUF/blob/main/Mistral-7B-Instruct-v0.3.IQ3_XS.gguf) | IQ3_XS | 3 | 2.7 GB| 3.7 GB | small, substantial quality loss | +| [Mistral-7B-Instruct-v0.3.IQ3_S.gguf](https://huggingface.co/CISCai/Mistral-7B-Instruct-v0.3-SOTA-GGUF/blob/main/Mistral-7B-Instruct-v0.3.IQ3_S.gguf) | IQ3_S | 3 | 2.8 GB| 3.8 GB | small, greater quality loss | +| [Mistral-7B-Instruct-v0.3.IQ3_M.gguf](https://huggingface.co/CISCai/Mistral-7B-Instruct-v0.3-SOTA-GGUF/blob/main/Mistral-7B-Instruct-v0.3.IQ3_M.gguf) | IQ3_M | 3 | 3.0 GB| 4.0 GB | medium, balanced quality - recommended | +| [Mistral-7B-Instruct-v0.3.IQ4_XS.gguf](https://huggingface.co/CISCai/Mistral-7B-Instruct-v0.3-SOTA-GGUF/blob/main/Mistral-7B-Instruct-v0.3.IQ4_XS.gguf) | IQ4_XS | 4 | 3.4 GB| 4.4 GB | small, substantial quality loss | + +Generated importance matrix file: [Mistral-7B-Instruct-v0.3.imatrix.dat](https://huggingface.co/CISCai/Mistral-7B-Instruct-v0.3-SOTA-GGUF/blob/main/Mistral-7B-Instruct-v0.3.imatrix.dat) + +**Note**: the above RAM figures assume no GPU offloading with 4K context. If layers are offloaded to the GPU, this will reduce RAM usage and use VRAM instead. + + + + +## Example `llama.cpp` command + +Make sure you are using `llama.cpp` from commit [0becb22](https://github.com/ggerganov/llama.cpp/commit/0becb22ac05b6542bd9d5f2235691aa1d3d4d307) or later. + +```shell +./main -ngl 33 -m Mistral-7B-Instruct-v0.3.IQ4_XS.gguf --color -c 32768 --temp 0 --repeat-penalty 1.1 -p "[AVAILABLE_TOOLS] {tools}[/AVAILABLE_TOOLS][INST] {prompt}[/INST]" +``` + +Change `-ngl 33` to the number of layers to offload to GPU. Remove it if you don't have GPU acceleration. + +Change `-c 32768` to the desired sequence length. + +If you want to have a chat-style conversation, replace the `-p ` argument with `-i -ins` + +If you are low on V/RAM try quantizing the K-cache with `-ctk q8_0` or even `-ctk q4_0` for big memory savings (depending on context size). +There is a similar option for V-cache (`-ctv`), however that is [not working yet](https://github.com/ggerganov/llama.cpp/issues/4425). + +For other parameters and how to use them, please refer to [the llama.cpp documentation](https://github.com/ggerganov/llama.cpp/blob/master/examples/main/README.md) + +## How to run from Python code + +You can use GGUF models from Python using the [llama-cpp-python](https://github.com/abetlen/llama-cpp-python) module. + +### How to load this model in Python code, using llama-cpp-python + +For full documentation, please see: [llama-cpp-python docs](https://llama-cpp-python.readthedocs.io/en/latest/). + +#### First install the package + +Run one of the following commands, according to your system: + +```shell +# Prebuilt wheel with basic CPU support +pip install llama-cpp-python --extra-index-url https://abetlen.github.io/llama-cpp-python/whl/cpu +# Prebuilt wheel with NVidia CUDA acceleration +pip install llama-cpp-python --extra-index-url https://abetlen.github.io/llama-cpp-python/whl/cu121 (or cu122 etc.) +# Prebuilt wheel with Metal GPU acceleration +pip install llama-cpp-python --extra-index-url https://abetlen.github.io/llama-cpp-python/whl/metal +# Build base version with no GPU acceleration +pip install llama-cpp-python +# With NVidia CUDA acceleration +CMAKE_ARGS="-DLLAMA_CUDA=on" pip install llama-cpp-python +# Or with OpenBLAS acceleration +CMAKE_ARGS="-DLLAMA_BLAS=ON -DLLAMA_BLAS_VENDOR=OpenBLAS" pip install llama-cpp-python +# Or with CLBLast acceleration +CMAKE_ARGS="-DLLAMA_CLBLAST=on" pip install llama-cpp-python +# Or with AMD ROCm GPU acceleration (Linux only) +CMAKE_ARGS="-DLLAMA_HIPBLAS=on" pip install llama-cpp-python +# Or with Metal GPU acceleration for macOS systems only +CMAKE_ARGS="-DLLAMA_METAL=on" pip install llama-cpp-python +# Or with Vulkan acceleration +CMAKE_ARGS="-DLLAMA_VULKAN=on" pip install llama-cpp-python +# Or with Kompute acceleration +CMAKE_ARGS="-DLLAMA_KOMPUTE=on" pip install llama-cpp-python +# Or with SYCL acceleration +CMAKE_ARGS="-DLLAMA_SYCL=on -DCMAKE_C_COMPILER=icx -DCMAKE_CXX_COMPILER=icpx" pip install llama-cpp-python + +# In windows, to set the variables CMAKE_ARGS in PowerShell, follow this format; eg for NVidia CUDA: +$env:CMAKE_ARGS = "-DLLAMA_CUDA=on" +pip install llama-cpp-python +``` + +#### Simple llama-cpp-python example code + +```python +from llama_cpp import Llama + +# Chat Completion API + +llm = Llama(model_path="./Mistral-7B-Instruct-v0.3.IQ4_XS.gguf", n_gpu_layers=33, n_ctx=32768) +print(llm.create_chat_completion( + messages = [ + { + "role": "user", + "content": "Pick a LeetCode challenge and solve it in Python." + } + ] +)) +``` + +#### Simple llama-cpp-python example function calling code + +```python +from llama_cpp import Llama + +# Chat Completion API + +grammar = LlamaGrammar.from_json_schema(json.dumps({ + "type": "array", + "items": { + "type": "object", + "required": [ "name", "arguments" ], + "properties": { + "name": { + "type": "string" + }, + "arguments": { + "type": "object" + } + } + } +})) + +llm = Llama(model_path="./Mistral-7B-Instruct-v0.3.IQ4_XS.gguf", n_gpu_layers=33, n_ctx=32768) +response = llm.create_chat_completion( + temperature = 0.0, + repeat_penalty = 1.1, + messages = [ + { + "role": "user", + "content": "What's the weather like in Oslo and Stockholm?" + } + ], + tools=[{ + "type": "function", + "function": { + "name": "get_current_weather", + "description": "Get the current weather in a given location", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA" + }, + "unit": { + "type": "string", + "enum": [ "celsius", "fahrenheit" ] + } + }, + "required": [ "location" ] + } + } + }], + grammar = grammar +) +print(json.loads(response["choices"][0]["text"])) + +print(llm.create_chat_completion( + temperature = 0.0, + repeat_penalty = 1.1, + messages = [ + { + "role": "user", + "content": "What's the weather like in Oslo?" + }, + { # The tool_calls is from the response to the above with tool_choice active + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call__0_get_current_weather_cmpl-...", + "type": "function", + "function": { + "name": "get_current_weather", + "arguments": '{ "location": "Oslo, NO" ,"unit": "celsius"} ' + } + } + ] + }, + { # The tool_call_id is from tool_calls and content is the result from the function call you made + "role": "tool", + "content": "20", + "tool_call_id": "call__0_get_current_weather_cmpl-..." + } + ], + tools=[{ + "type": "function", + "function": { + "name": "get_current_weather", + "description": "Get the current weather in a given location", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA" + }, + "unit": { + "type": "string", + "enum": [ "celsius", "fahrenheit" ] + } + }, + "required": [ "location" ] + } + } + }], + #tool_choice={ + # "type": "function", + # "function": { + # "name": "get_current_weather" + # } + #} +)) +``` + +