初始化项目，由ModelHub XC社区提供模型
Model: CISCai/Mistral-7B-Instruct-v0.3-SOTA-GGUF Source: Original Platform
This commit is contained in:
ModelHub XC
48
.gitattributes
vendored
Normal file
48
.gitattributes
vendored
Normal file
@@ -0,0 +1,48 @@
|
||||
*.7z filter=lfs diff=lfs merge=lfs -text
|
||||
*.arrow filter=lfs diff=lfs merge=lfs -text
|
||||
*.bin filter=lfs diff=lfs merge=lfs -text
|
||||
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
||||
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
||||
*.ftz filter=lfs diff=lfs merge=lfs -text
|
||||
*.gz filter=lfs diff=lfs merge=lfs -text
|
||||
*.h5 filter=lfs diff=lfs merge=lfs -text
|
||||
*.joblib filter=lfs diff=lfs merge=lfs -text
|
||||
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
||||
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
||||
*.model filter=lfs diff=lfs merge=lfs -text
|
||||
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
||||
*.npy filter=lfs diff=lfs merge=lfs -text
|
||||
*.npz filter=lfs diff=lfs merge=lfs -text
|
||||
*.onnx filter=lfs diff=lfs merge=lfs -text
|
||||
*.ot filter=lfs diff=lfs merge=lfs -text
|
||||
*.parquet filter=lfs diff=lfs merge=lfs -text
|
||||
*.pb filter=lfs diff=lfs merge=lfs -text
|
||||
*.pickle filter=lfs diff=lfs merge=lfs -text
|
||||
*.pkl filter=lfs diff=lfs merge=lfs -text
|
||||
*.pt filter=lfs diff=lfs merge=lfs -text
|
||||
*.pth filter=lfs diff=lfs merge=lfs -text
|
||||
*.rar filter=lfs diff=lfs merge=lfs -text
|
||||
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
||||
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
||||
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
||||
*.tar filter=lfs diff=lfs merge=lfs -text
|
||||
*.tflite filter=lfs diff=lfs merge=lfs -text
|
||||
*.tgz filter=lfs diff=lfs merge=lfs -text
|
||||
*.wasm filter=lfs diff=lfs merge=lfs -text
|
||||
*.xz filter=lfs diff=lfs merge=lfs -text
|
||||
*.zip filter=lfs diff=lfs merge=lfs -text
|
||||
*.zst filter=lfs diff=lfs merge=lfs -text
|
||||
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
||||
Mistral-7B-Instruct-v0.3.fp16.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
Mistral-7B-Instruct-v0.3.imatrix.dat filter=lfs diff=lfs merge=lfs -text
|
||||
Mistral-7B-Instruct-v0.3.IQ1_M.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
Mistral-7B-Instruct-v0.3.IQ1_S.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
Mistral-7B-Instruct-v0.3.IQ2_M.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
Mistral-7B-Instruct-v0.3.IQ2_S.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
Mistral-7B-Instruct-v0.3.IQ2_XS.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
Mistral-7B-Instruct-v0.3.IQ2_XXS.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
Mistral-7B-Instruct-v0.3.IQ3_M.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
Mistral-7B-Instruct-v0.3.IQ3_S.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
Mistral-7B-Instruct-v0.3.IQ3_XS.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
Mistral-7B-Instruct-v0.3.IQ3_XXS.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
Mistral-7B-Instruct-v0.3.IQ4_XS.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
3
Mistral-7B-Instruct-v0.3.IQ1_M.gguf
Normal file
3
Mistral-7B-Instruct-v0.3.IQ1_M.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:6d1a2dd797732b2bd81522c0dc2ce07f02c9a760f760e16302dda6477d1b7b59
|
||||
size 1757664288
|
||||
3
Mistral-7B-Instruct-v0.3.IQ1_S.gguf
Normal file
3
Mistral-7B-Instruct-v0.3.IQ1_S.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:23a3cfd34d22263ccf49dfc02e7f3ea7c983118680a9dedca31817efdbd3efab
|
||||
size 1615320096
|
||||
3
Mistral-7B-Instruct-v0.3.IQ2_M.gguf
Normal file
3
Mistral-7B-Instruct-v0.3.IQ2_M.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:7e93031a8c059e014a904b2f8cebdf073ff7310f9a6d2579c07c8d6ebdc6d8ae
|
||||
size 2504250400
|
||||
3
Mistral-7B-Instruct-v0.3.IQ2_S.gguf
Normal file
3
Mistral-7B-Instruct-v0.3.IQ2_S.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:2afeb6b4536c43014c5575aed66ce0577c965b9bf5019bc7c43cdbbaf520bc4a
|
||||
size 2314458144
|
||||
3
Mistral-7B-Instruct-v0.3.IQ2_XS.gguf
Normal file
3
Mistral-7B-Instruct-v0.3.IQ2_XS.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:4ff8d8d252da2955e69420ddf6ba79c7f9364c52a269d3333edefb6ca4e58bac
|
||||
size 2201474080
|
||||
3
Mistral-7B-Instruct-v0.3.IQ2_XXS.gguf
Normal file
3
Mistral-7B-Instruct-v0.3.IQ2_XXS.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:9ebea120c9f96af5595fbb1b7addc8daa7841bfcf16703d0382d21b22e66a8e5
|
||||
size 1994904608
|
||||
3
Mistral-7B-Instruct-v0.3.IQ3_M.gguf
Normal file
3
Mistral-7B-Instruct-v0.3.IQ3_M.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:8d49160a4dbf6af69ab8a22bcee90d821e724aa0deddd63c2625f60eaeff4af4
|
||||
size 3288847392
|
||||
3
Mistral-7B-Instruct-v0.3.IQ3_S.gguf
Normal file
3
Mistral-7B-Instruct-v0.3.IQ3_S.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:156cbd42ee5a635bb795dc36bd3283d7c746b2e12bfcd451163756a283345e0c
|
||||
size 3186349088
|
||||
3
Mistral-7B-Instruct-v0.3.IQ3_XS.gguf
Normal file
3
Mistral-7B-Instruct-v0.3.IQ3_XS.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:84985ba223dc145565068e1d1b617cc44201c46ad87b009c7281e708613bf484
|
||||
size 3022771232
|
||||
3
Mistral-7B-Instruct-v0.3.IQ3_XXS.gguf
Normal file
3
Mistral-7B-Instruct-v0.3.IQ3_XXS.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:26233eb5e3fc54bbdfe977934ef7b015dd5dac759e424710f024fc795781027c
|
||||
size 2830881824
|
||||
3
Mistral-7B-Instruct-v0.3.IQ4_XS.gguf
Normal file
3
Mistral-7B-Instruct-v0.3.IQ4_XS.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:c0afba232e1e46573db08bd965c82ef583f0b71131be840a19982339d94e067c
|
||||
size 3911963680
|
||||
3
Mistral-7B-Instruct-v0.3.fp16.gguf
Normal file
3
Mistral-7B-Instruct-v0.3.fp16.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:dbd62e09c7201d790605d3a8be94260bfbecd70c470ed297e4d584fa319ed8ab
|
||||
size 14497338144
|
||||
3
Mistral-7B-Instruct-v0.3.imatrix.dat
Normal file
3
Mistral-7B-Instruct-v0.3.imatrix.dat
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:e44d933bda5232dd9081c20e0ae2ef2df5aab5c4388fc308b237d1454e694e81
|
||||
size 4988162
|
||||
290
README.md
Normal file
290
README.md
Normal file
@@ -0,0 +1,290 @@
|
||||
---
|
||||
base_model: mistralai/Mistral-7B-Instruct-v0.3
|
||||
language:
|
||||
- en
|
||||
pipeline_tag: text-generation
|
||||
license: apache-2.0
|
||||
model_creator: Mistral AI
|
||||
model_name: Mistral-7B-Instruct-v0.3
|
||||
model_type: mistral
|
||||
quantized_by: CISC
|
||||
---
|
||||
|
||||
# Mistral-7B-Instruct-v0.3 - SOTA GGUF
|
||||
- Model creator: [Mistral AI](https://huggingface.co/mistralai)
|
||||
- Original model: [Mistral-7B-Instruct-v0.3](https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.3)
|
||||
|
||||
<!-- description start -->
|
||||
## Description
|
||||
|
||||
This repo contains State Of The Art quantized GGUF format model files for [Mistral-7B-Instruct-v0.3](https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.3).
|
||||
|
||||
Quantization was done with an importance matrix that was trained for ~1M tokens (256 batches of 4096 tokens) of [groups_merged.txt](https://github.com/ggerganov/llama.cpp/discussions/5263#discussioncomment-8395384) and [wiki.train.raw](https://raw.githubusercontent.com/pytorch/examples/main/word_language_model/data/wikitext-2/train.txt) concatenated.
|
||||
|
||||
The embedded chat template has been extended to support function calling via OpenAI-compatible `tools` parameter, see [example](#simple-llama-cpp-python-example-function-calling-code).
|
||||
|
||||
<!-- description end -->
|
||||
|
||||
|
||||
<!-- prompt-template start -->
|
||||
## Prompt template: Mistral v3
|
||||
|
||||
```
|
||||
[AVAILABLE_TOOLS] [{"name": "function_name", "description": "Description", "parameters": {...}}, ...][/AVAILABLE_TOOLS][INST] {prompt}[/INST]
|
||||
```
|
||||
|
||||
<!-- prompt-template end -->
|
||||
|
||||
|
||||
<!-- compatibility_gguf start -->
|
||||
## Compatibility
|
||||
|
||||
These quantised GGUFv3 files are compatible with llama.cpp from February 27th 2024 onwards, as of commit [0becb22](https://github.com/ggerganov/llama.cpp/commit/0becb22ac05b6542bd9d5f2235691aa1d3d4d307)
|
||||
|
||||
They are also compatible with many third party UIs and libraries provided they are built using a recent llama.cpp.
|
||||
|
||||
## Explanation of quantisation methods
|
||||
|
||||
<details>
|
||||
<summary>Click to see details</summary>
|
||||
|
||||
The new methods available are:
|
||||
|
||||
* GGML_TYPE_IQ1_S - 1-bit quantization in super-blocks with an importance matrix applied, effectively using 1.56 bits per weight (bpw)
|
||||
* GGML_TYPE_IQ1_M - 1-bit quantization in super-blocks with an importance matrix applied, effectively using 1.75 bpw
|
||||
* GGML_TYPE_IQ2_XXS - 2-bit quantization in super-blocks with an importance matrix applied, effectively using 2.06 bpw
|
||||
* GGML_TYPE_IQ2_XS - 2-bit quantization in super-blocks with an importance matrix applied, effectively using 2.31 bpw
|
||||
* GGML_TYPE_IQ2_S - 2-bit quantization in super-blocks with an importance matrix applied, effectively using 2.5 bpw
|
||||
* GGML_TYPE_IQ2_M - 2-bit quantization in super-blocks with an importance matrix applied, effectively using 2.7 bpw
|
||||
* GGML_TYPE_IQ3_XXS - 3-bit quantization in super-blocks with an importance matrix applied, effectively using 3.06 bpw
|
||||
* GGML_TYPE_IQ3_XS - 3-bit quantization in super-blocks with an importance matrix applied, effectively using 3.3 bpw
|
||||
* GGML_TYPE_IQ3_S - 3-bit quantization in super-blocks with an importance matrix applied, effectively using 3.44 bpw
|
||||
* GGML_TYPE_IQ3_M - 3-bit quantization in super-blocks with an importance matrix applied, effectively using 3.66 bpw
|
||||
* GGML_TYPE_IQ4_XS - 4-bit quantization in super-blocks with an importance matrix applied, effectively using 4.25 bpw
|
||||
* GGML_TYPE_IQ4_NL - 4-bit non-linearly mapped quantization with an importance matrix applied, effectively using 4.5 bpw
|
||||
|
||||
Refer to the Provided Files table below to see what files use which methods, and how.
|
||||
</details>
|
||||
<!-- compatibility_gguf end -->
|
||||
|
||||
<!-- README_GGUF.md-provided-files start -->
|
||||
## Provided files
|
||||
|
||||
| Name | Quant method | Bits | Size | Max RAM required | Use case |
|
||||
| ---- | ---- | ---- | ---- | ---- | ----- |
|
||||
| [Mistral-7B-Instruct-v0.3.IQ1_S.gguf](https://huggingface.co/CISCai/Mistral-7B-Instruct-v0.3-SOTA-GGUF/blob/main/Mistral-7B-Instruct-v0.3.IQ1_S.gguf) | IQ1_S | 1 | 1.5 GB| 2.5 GB | smallest, significant quality loss - **TBD**: Waiting for [this issue](https://github.com/ggerganov/llama.cpp/issues/5996) to be resolved |
|
||||
| [Mistral-7B-Instruct-v0.3.IQ1_M.gguf](https://huggingface.co/CISCai/Mistral-7B-Instruct-v0.3-SOTA-GGUF/blob/main/Mistral-7B-Instruct-v0.3.IQ1_M.gguf) | IQ1_M | 1 | 1.6 GB| 2.6 GB | very small, significant quality loss |
|
||||
| [Mistral-7B-Instruct-v0.3.IQ2_XXS.gguf](https://huggingface.co/CISCai/Mistral-7B-Instruct-v0.3-SOTA-GGUF/blob/main/Mistral-7B-Instruct-v0.3.IQ2_XXS.gguf) | IQ2_XXS | 2 | 1.8 GB| 2.8 GB | very small, high quality loss |
|
||||
| [Mistral-7B-Instruct-v0.3.IQ2_XS.gguf](https://huggingface.co/CISCai/Mistral-7B-Instruct-v0.3-SOTA-GGUF/blob/main/Mistral-7B-Instruct-v0.3.IQ2_XS.gguf) | IQ2_XS | 2 | 1.9 GB| 2.9 GB | very small, high quality loss |
|
||||
| [Mistral-7B-Instruct-v0.3.IQ2_S.gguf](https://huggingface.co/CISCai/Mistral-7B-Instruct-v0.3-SOTA-GGUF/blob/main/Mistral-7B-Instruct-v0.3.IQ2_S.gguf) | IQ2_S | 2 | 2.1 GB| 3.1 GB | small, substantial quality loss |
|
||||
| [Mistral-7B-Instruct-v0.3.IQ2_M.gguf](https://huggingface.co/CISCai/Mistral-7B-Instruct-v0.3-SOTA-GGUF/blob/main/Mistral-7B-Instruct-v0.3.IQ2_M.gguf) | IQ2_M | 2 | 2.2 GB| 3.2 GB | small, greater quality loss |
|
||||
| [Mistral-7B-Instruct-v0.3.IQ3_XXS.gguf](https://huggingface.co/CISCai/Mistral-7B-Instruct-v0.3-SOTA-GGUF/blob/main/Mistral-7B-Instruct-v0.3.IQ3_XXS.gguf) | IQ3_XXS | 3 | 2.5 GB| 3.5 GB | very small, high quality loss |
|
||||
| [Mistral-7B-Instruct-v0.3.IQ3_XS.gguf](https://huggingface.co/CISCai/Mistral-7B-Instruct-v0.3-SOTA-GGUF/blob/main/Mistral-7B-Instruct-v0.3.IQ3_XS.gguf) | IQ3_XS | 3 | 2.7 GB| 3.7 GB | small, substantial quality loss |
|
||||
| [Mistral-7B-Instruct-v0.3.IQ3_S.gguf](https://huggingface.co/CISCai/Mistral-7B-Instruct-v0.3-SOTA-GGUF/blob/main/Mistral-7B-Instruct-v0.3.IQ3_S.gguf) | IQ3_S | 3 | 2.8 GB| 3.8 GB | small, greater quality loss |
|
||||
| [Mistral-7B-Instruct-v0.3.IQ3_M.gguf](https://huggingface.co/CISCai/Mistral-7B-Instruct-v0.3-SOTA-GGUF/blob/main/Mistral-7B-Instruct-v0.3.IQ3_M.gguf) | IQ3_M | 3 | 3.0 GB| 4.0 GB | medium, balanced quality - recommended |
|
||||
| [Mistral-7B-Instruct-v0.3.IQ4_XS.gguf](https://huggingface.co/CISCai/Mistral-7B-Instruct-v0.3-SOTA-GGUF/blob/main/Mistral-7B-Instruct-v0.3.IQ4_XS.gguf) | IQ4_XS | 4 | 3.4 GB| 4.4 GB | small, substantial quality loss |
|
||||
|
||||
Generated importance matrix file: [Mistral-7B-Instruct-v0.3.imatrix.dat](https://huggingface.co/CISCai/Mistral-7B-Instruct-v0.3-SOTA-GGUF/blob/main/Mistral-7B-Instruct-v0.3.imatrix.dat)
|
||||
|
||||
**Note**: the above RAM figures assume no GPU offloading with 4K context. If layers are offloaded to the GPU, this will reduce RAM usage and use VRAM instead.
|
||||
|
||||
<!-- README_GGUF.md-provided-files end -->
|
||||
|
||||
<!-- README_GGUF.md-how-to-run start -->
|
||||
## Example `llama.cpp` command
|
||||
|
||||
Make sure you are using `llama.cpp` from commit [0becb22](https://github.com/ggerganov/llama.cpp/commit/0becb22ac05b6542bd9d5f2235691aa1d3d4d307) or later.
|
||||
|
||||
```shell
|
||||
./main -ngl 33 -m Mistral-7B-Instruct-v0.3.IQ4_XS.gguf --color -c 32768 --temp 0 --repeat-penalty 1.1 -p "[AVAILABLE_TOOLS] {tools}[/AVAILABLE_TOOLS][INST] {prompt}[/INST]"
|
||||
```
|
||||
|
||||
Change `-ngl 33` to the number of layers to offload to GPU. Remove it if you don't have GPU acceleration.
|
||||
|
||||
Change `-c 32768` to the desired sequence length.
|
||||
|
||||
If you want to have a chat-style conversation, replace the `-p <PROMPT>` argument with `-i -ins`
|
||||
|
||||
If you are low on V/RAM try quantizing the K-cache with `-ctk q8_0` or even `-ctk q4_0` for big memory savings (depending on context size).
|
||||
There is a similar option for V-cache (`-ctv`), however that is [not working yet](https://github.com/ggerganov/llama.cpp/issues/4425).
|
||||
|
||||
For other parameters and how to use them, please refer to [the llama.cpp documentation](https://github.com/ggerganov/llama.cpp/blob/master/examples/main/README.md)
|
||||
|
||||
## How to run from Python code
|
||||
|
||||
You can use GGUF models from Python using the [llama-cpp-python](https://github.com/abetlen/llama-cpp-python) module.
|
||||
|
||||
### How to load this model in Python code, using llama-cpp-python
|
||||
|
||||
For full documentation, please see: [llama-cpp-python docs](https://llama-cpp-python.readthedocs.io/en/latest/).
|
||||
|
||||
#### First install the package
|
||||
|
||||
Run one of the following commands, according to your system:
|
||||
|
||||
```shell
|
||||
# Prebuilt wheel with basic CPU support
|
||||
pip install llama-cpp-python --extra-index-url https://abetlen.github.io/llama-cpp-python/whl/cpu
|
||||
# Prebuilt wheel with NVidia CUDA acceleration
|
||||
pip install llama-cpp-python --extra-index-url https://abetlen.github.io/llama-cpp-python/whl/cu121 (or cu122 etc.)
|
||||
# Prebuilt wheel with Metal GPU acceleration
|
||||
pip install llama-cpp-python --extra-index-url https://abetlen.github.io/llama-cpp-python/whl/metal
|
||||
# Build base version with no GPU acceleration
|
||||
pip install llama-cpp-python
|
||||
# With NVidia CUDA acceleration
|
||||
CMAKE_ARGS="-DLLAMA_CUDA=on" pip install llama-cpp-python
|
||||
# Or with OpenBLAS acceleration
|
||||
CMAKE_ARGS="-DLLAMA_BLAS=ON -DLLAMA_BLAS_VENDOR=OpenBLAS" pip install llama-cpp-python
|
||||
# Or with CLBLast acceleration
|
||||
CMAKE_ARGS="-DLLAMA_CLBLAST=on" pip install llama-cpp-python
|
||||
# Or with AMD ROCm GPU acceleration (Linux only)
|
||||
CMAKE_ARGS="-DLLAMA_HIPBLAS=on" pip install llama-cpp-python
|
||||
# Or with Metal GPU acceleration for macOS systems only
|
||||
CMAKE_ARGS="-DLLAMA_METAL=on" pip install llama-cpp-python
|
||||
# Or with Vulkan acceleration
|
||||
CMAKE_ARGS="-DLLAMA_VULKAN=on" pip install llama-cpp-python
|
||||
# Or with Kompute acceleration
|
||||
CMAKE_ARGS="-DLLAMA_KOMPUTE=on" pip install llama-cpp-python
|
||||
# Or with SYCL acceleration
|
||||
CMAKE_ARGS="-DLLAMA_SYCL=on -DCMAKE_C_COMPILER=icx -DCMAKE_CXX_COMPILER=icpx" pip install llama-cpp-python
|
||||
|
||||
# In windows, to set the variables CMAKE_ARGS in PowerShell, follow this format; eg for NVidia CUDA:
|
||||
$env:CMAKE_ARGS = "-DLLAMA_CUDA=on"
|
||||
pip install llama-cpp-python
|
||||
```
|
||||
|
||||
#### Simple llama-cpp-python example code
|
||||
|
||||
```python
|
||||
from llama_cpp import Llama
|
||||
|
||||
# Chat Completion API
|
||||
|
||||
llm = Llama(model_path="./Mistral-7B-Instruct-v0.3.IQ4_XS.gguf", n_gpu_layers=33, n_ctx=32768)
|
||||
print(llm.create_chat_completion(
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Pick a LeetCode challenge and solve it in Python."
|
||||
}
|
||||
]
|
||||
))
|
||||
```
|
||||
|
||||
#### Simple llama-cpp-python example function calling code
|
||||
|
||||
```python
|
||||
from llama_cpp import Llama
|
||||
|
||||
# Chat Completion API
|
||||
|
||||
grammar = LlamaGrammar.from_json_schema(json.dumps({
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"required": [ "name", "arguments" ],
|
||||
"properties": {
|
||||
"name": {
|
||||
"type": "string"
|
||||
},
|
||||
"arguments": {
|
||||
"type": "object"
|
||||
}
|
||||
}
|
||||
}
|
||||
}))
|
||||
|
||||
llm = Llama(model_path="./Mistral-7B-Instruct-v0.3.IQ4_XS.gguf", n_gpu_layers=33, n_ctx=32768)
|
||||
response = llm.create_chat_completion(
|
||||
temperature = 0.0,
|
||||
repeat_penalty = 1.1,
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "What's the weather like in Oslo and Stockholm?"
|
||||
}
|
||||
],
|
||||
tools=[{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_current_weather",
|
||||
"description": "Get the current weather in a given location",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"location": {
|
||||
"type": "string",
|
||||
"description": "The city and state, e.g. San Francisco, CA"
|
||||
},
|
||||
"unit": {
|
||||
"type": "string",
|
||||
"enum": [ "celsius", "fahrenheit" ]
|
||||
}
|
||||
},
|
||||
"required": [ "location" ]
|
||||
}
|
||||
}
|
||||
}],
|
||||
grammar = grammar
|
||||
)
|
||||
print(json.loads(response["choices"][0]["text"]))
|
||||
|
||||
print(llm.create_chat_completion(
|
||||
temperature = 0.0,
|
||||
repeat_penalty = 1.1,
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "What's the weather like in Oslo?"
|
||||
},
|
||||
{ # The tool_calls is from the response to the above with tool_choice active
|
||||
"role": "assistant",
|
||||
"content": None,
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "call__0_get_current_weather_cmpl-...",
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_current_weather",
|
||||
"arguments": '{ "location": "Oslo, NO" ,"unit": "celsius"} '
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{ # The tool_call_id is from tool_calls and content is the result from the function call you made
|
||||
"role": "tool",
|
||||
"content": "20",
|
||||
"tool_call_id": "call__0_get_current_weather_cmpl-..."
|
||||
}
|
||||
],
|
||||
tools=[{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_current_weather",
|
||||
"description": "Get the current weather in a given location",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"location": {
|
||||
"type": "string",
|
||||
"description": "The city and state, e.g. San Francisco, CA"
|
||||
},
|
||||
"unit": {
|
||||
"type": "string",
|
||||
"enum": [ "celsius", "fahrenheit" ]
|
||||
}
|
||||
},
|
||||
"required": [ "location" ]
|
||||
}
|
||||
}
|
||||
}],
|
||||
#tool_choice={
|
||||
# "type": "function",
|
||||
# "function": {
|
||||
# "name": "get_current_weather"
|
||||
# }
|
||||
#}
|
||||
))
|
||||
```
|
||||
|
||||
<!-- README_GGUF.md-how-to-run end -->
|
||||
Reference in New Issue
Block a user