初始化项目，由ModelHub XC社区提供模型
Model: legraphista/NuminaMath-7B-TIR-IMat-GGUF Source: Original Platform
This commit is contained in:
ModelHub XC
61
.gitattributes
vendored
Normal file
61
.gitattributes
vendored
Normal file
@@ -0,0 +1,61 @@
|
||||
*.7z filter=lfs diff=lfs merge=lfs -text
|
||||
*.arrow filter=lfs diff=lfs merge=lfs -text
|
||||
*.bin filter=lfs diff=lfs merge=lfs -text
|
||||
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
||||
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
||||
*.ftz filter=lfs diff=lfs merge=lfs -text
|
||||
*.gz filter=lfs diff=lfs merge=lfs -text
|
||||
*.h5 filter=lfs diff=lfs merge=lfs -text
|
||||
*.joblib filter=lfs diff=lfs merge=lfs -text
|
||||
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
||||
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
||||
*.model filter=lfs diff=lfs merge=lfs -text
|
||||
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
||||
*.npy filter=lfs diff=lfs merge=lfs -text
|
||||
*.npz filter=lfs diff=lfs merge=lfs -text
|
||||
*.onnx filter=lfs diff=lfs merge=lfs -text
|
||||
*.ot filter=lfs diff=lfs merge=lfs -text
|
||||
*.parquet filter=lfs diff=lfs merge=lfs -text
|
||||
*.pb filter=lfs diff=lfs merge=lfs -text
|
||||
*.pickle filter=lfs diff=lfs merge=lfs -text
|
||||
*.pkl filter=lfs diff=lfs merge=lfs -text
|
||||
*.pt filter=lfs diff=lfs merge=lfs -text
|
||||
*.pth filter=lfs diff=lfs merge=lfs -text
|
||||
*.rar filter=lfs diff=lfs merge=lfs -text
|
||||
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
||||
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
||||
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
||||
*.tar filter=lfs diff=lfs merge=lfs -text
|
||||
*.tflite filter=lfs diff=lfs merge=lfs -text
|
||||
*.tgz filter=lfs diff=lfs merge=lfs -text
|
||||
*.wasm filter=lfs diff=lfs merge=lfs -text
|
||||
*.xz filter=lfs diff=lfs merge=lfs -text
|
||||
*.zip filter=lfs diff=lfs merge=lfs -text
|
||||
*.zst filter=lfs diff=lfs merge=lfs -text
|
||||
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
||||
imatrix.dat filter=lfs diff=lfs merge=lfs -text
|
||||
NuminaMath-7B-TIR.Q6_K.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
NuminaMath-7B-TIR.Q8_0.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
NuminaMath-7B-TIR.Q5_K.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
NuminaMath-7B-TIR.BF16.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
NuminaMath-7B-TIR.Q5_K_S.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
NuminaMath-7B-TIR.Q4_K.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
NuminaMath-7B-TIR.FP16.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
NuminaMath-7B-TIR.Q3_K.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
NuminaMath-7B-TIR.Q2_K.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
NuminaMath-7B-TIR.Q4_K_S.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
NuminaMath-7B-TIR.Q3_K_L.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
NuminaMath-7B-TIR.Q3_K_S.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
NuminaMath-7B-TIR.Q2_K_S.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
NuminaMath-7B-TIR.IQ4_NL.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
NuminaMath-7B-TIR.IQ4_XS.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
NuminaMath-7B-TIR.IQ3_M.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
NuminaMath-7B-TIR.IQ3_XS.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
NuminaMath-7B-TIR.IQ3_S.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
NuminaMath-7B-TIR.IQ3_XXS.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
NuminaMath-7B-TIR.IQ2_M.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
NuminaMath-7B-TIR.IQ2_S.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
NuminaMath-7B-TIR.IQ2_XS.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
NuminaMath-7B-TIR.IQ2_XXS.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
NuminaMath-7B-TIR.IQ1_M.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
NuminaMath-7B-TIR.IQ1_S.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
3
NuminaMath-7B-TIR.BF16.gguf
Normal file
3
NuminaMath-7B-TIR.BF16.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:51e5302d3c5d646288606770d33692374b43dab68540089df5050b8a34f042bc
|
||||
size 13825218080
|
||||
3
NuminaMath-7B-TIR.FP16.gguf
Normal file
3
NuminaMath-7B-TIR.FP16.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:05fff9d7a0fc4a511c5ae0356654dd60b0e3ab16e7f53cf6a850d520b2974cce
|
||||
size 13825218080
|
||||
3
NuminaMath-7B-TIR.IQ1_M.gguf
Normal file
3
NuminaMath-7B-TIR.IQ1_M.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:f2768707a9be754e77279110239da2dbe98abc3e8df0ac19ada6d968bb785648
|
||||
size 1848563488
|
||||
3
NuminaMath-7B-TIR.IQ1_S.gguf
Normal file
3
NuminaMath-7B-TIR.IQ1_S.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:ea569a543dc5bbddc5de83819ce408b2a0d2e3b5e5223878a1dbeccd35963306
|
||||
size 1733031712
|
||||
3
NuminaMath-7B-TIR.IQ2_M.gguf
Normal file
3
NuminaMath-7B-TIR.IQ2_M.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:2785ddee3197df31cd5d0c1c23907c688c79bf74d0e5fb1a0a673ce21dee2fe7
|
||||
size 2543163168
|
||||
3
NuminaMath-7B-TIR.IQ2_S.gguf
Normal file
3
NuminaMath-7B-TIR.IQ2_S.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:572db451928955463d3a6a5abd53f95198f84a2a8b9b166f8f7131c720de550f
|
||||
size 2389120800
|
||||
3
NuminaMath-7B-TIR.IQ2_XS.gguf
Normal file
3
NuminaMath-7B-TIR.IQ2_XS.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:80f376fc38afd6af14bd25875d92abab3b01fdf5a2ca5b33b8145510a59d2d85
|
||||
size 2210887456
|
||||
3
NuminaMath-7B-TIR.IQ2_XXS.gguf
Normal file
3
NuminaMath-7B-TIR.IQ2_XXS.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:28aaede532e417ae7956721e18e88b9ce01b5d6a932e98489bdfee1dc5c71047
|
||||
size 2041116448
|
||||
3
NuminaMath-7B-TIR.IQ3_M.gguf
Normal file
3
NuminaMath-7B-TIR.IQ3_M.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:591fdeef2f42d26dcfa82a546fa5f6154260d0e5400e95c9fef50f7ba66f2f7d
|
||||
size 3289675552
|
||||
3
NuminaMath-7B-TIR.IQ3_S.gguf
Normal file
3
NuminaMath-7B-TIR.IQ3_S.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:08c9528223c01ce631f812cb64fa31c9c405f69d8aff88090859ee81c48ab548
|
||||
size 3138017056
|
||||
3
NuminaMath-7B-TIR.IQ3_XS.gguf
Normal file
3
NuminaMath-7B-TIR.IQ3_XS.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:3fc27f1f5dc2dbf137411a1c9b4d82a8229a2b3c3d5e060414a51268bf58ccf8
|
||||
size 2993608480
|
||||
3
NuminaMath-7B-TIR.IQ3_XXS.gguf
Normal file
3
NuminaMath-7B-TIR.IQ3_XXS.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:ba852b185aee5e9bf10e8a37583bb3f57cf2c41bd237f91303825c53c42e0dbe
|
||||
size 2758399776
|
||||
3
NuminaMath-7B-TIR.IQ4_NL.gguf
Normal file
3
NuminaMath-7B-TIR.IQ4_NL.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:ea3faef52eb2d884321cae3fa673a11461ef42cde3e4a4e0b64cbe6f339b8db5
|
||||
size 4000061216
|
||||
3
NuminaMath-7B-TIR.IQ4_XS.gguf
Normal file
3
NuminaMath-7B-TIR.IQ4_XS.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:37ebdbbceba668e37bd45af0cea4910c1a287ccaa488bec79d128c908a9db229
|
||||
size 3797227296
|
||||
3
NuminaMath-7B-TIR.Q2_K.gguf
Normal file
3
NuminaMath-7B-TIR.Q2_K.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:9da0f0f0deb5ba20d636124086ad9afed1a09b5cfae8b22c23aba19c73968d76
|
||||
size 2718422816
|
||||
3
NuminaMath-7B-TIR.Q2_K_S.gguf
Normal file
3
NuminaMath-7B-TIR.Q2_K_S.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:5c910b6bf02f505dac87062bc0d3c491344f592f610619fc5769e68f18d63d5d
|
||||
size 2510509856
|
||||
3
NuminaMath-7B-TIR.Q3_K.gguf
Normal file
3
NuminaMath-7B-TIR.Q3_K.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:81b496c376cb5b7ca5649fe07914f06fbdd88d866ebcace67d2913ea7bcce09e
|
||||
size 3461191456
|
||||
3
NuminaMath-7B-TIR.Q3_K_L.gguf
Normal file
3
NuminaMath-7B-TIR.Q3_K_L.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:3efd59a428289030f6180f87b283aec0158cb82a373612fce151b577aa07de01
|
||||
size 3746273056
|
||||
3
NuminaMath-7B-TIR.Q3_K_S.gguf
Normal file
3
NuminaMath-7B-TIR.Q3_K_S.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:8d269026d57751db830b5d607c01589afcb1d160c1af958b7f6aa3d6915352c5
|
||||
size 3138017056
|
||||
3
NuminaMath-7B-TIR.Q4_K.gguf
Normal file
3
NuminaMath-7B-TIR.Q4_K.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:1990e2caf6aaadefde60cab049001a7645b5aa9e0aa6e5c7a0e9b540378a7f6b
|
||||
size 4223358752
|
||||
3
NuminaMath-7B-TIR.Q4_K_S.gguf
Normal file
3
NuminaMath-7B-TIR.Q4_K_S.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:e0785adfa4164a3a85c3548e08b9ae599efed0992153a316a98bed08ddf0cbc0
|
||||
size 4025358112
|
||||
3
NuminaMath-7B-TIR.Q5_K.gguf
Normal file
3
NuminaMath-7B-TIR.Q5_K.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:08e35fe8f8d4b9d37c1bf394c3e9af190ff21e6488b7a257ff3a49e6eced34ff
|
||||
size 4926428704
|
||||
3
NuminaMath-7B-TIR.Q5_K_S.gguf
Normal file
3
NuminaMath-7B-TIR.Q5_K_S.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:93aeefb0e74bbc8fb781fc00036bb839f0479ece24e19c68bad5e185157b67c9
|
||||
size 4811396640
|
||||
3
NuminaMath-7B-TIR.Q6_K.gguf
Normal file
3
NuminaMath-7B-TIR.Q6_K.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:3441fbaff49bda83d9678b8018af22877c0147c923454c2ba74fd231c93e3dbe
|
||||
size 5673440800
|
||||
3
NuminaMath-7B-TIR.Q8_0.gguf
Normal file
3
NuminaMath-7B-TIR.Q8_0.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:b87761460641f7fa0b78b5aeafa9b12ee37d2ff43410d1426bba1db6685d40a8
|
||||
size 7346984480
|
||||
148
README.md
Normal file
148
README.md
Normal file
@@ -0,0 +1,148 @@
|
||||
---
|
||||
base_model: AI-MO/NuminaMath-7B-TIR
|
||||
inference: false
|
||||
library_name: gguf
|
||||
license: apache-2.0
|
||||
model-index:
|
||||
- name: NuminaMath-7B-TIR
|
||||
results: []
|
||||
pipeline_tag: text-generation
|
||||
quantized_by: legraphista
|
||||
tags:
|
||||
- alignment-handbook
|
||||
- generated_from_trainer
|
||||
- quantized
|
||||
- GGUF
|
||||
- quantization
|
||||
- imat
|
||||
- imatrix
|
||||
- static
|
||||
- 16bit
|
||||
- 8bit
|
||||
- 6bit
|
||||
- 5bit
|
||||
- 4bit
|
||||
- 3bit
|
||||
- 2bit
|
||||
- 1bit
|
||||
---
|
||||
|
||||
# NuminaMath-7B-TIR-IMat-GGUF
|
||||
_Llama.cpp imatrix quantization of AI-MO/NuminaMath-7B-TIR_
|
||||
|
||||
Original Model: [AI-MO/NuminaMath-7B-TIR](https://huggingface.co/AI-MO/NuminaMath-7B-TIR)
|
||||
Original dtype: `BF16` (`bfloat16`)
|
||||
Quantized by: llama.cpp [b3371](https://github.com/ggerganov/llama.cpp/releases/tag/b3371)
|
||||
IMatrix dataset: [here](https://gist.githubusercontent.com/bartowski1182/eb213dccb3571f863da82e99418f81e8/raw/b2869d80f5c16fd7082594248e80144677736635/calibration_datav3.txt)
|
||||
|
||||
- [Files](#files)
|
||||
- [IMatrix](#imatrix)
|
||||
- [Common Quants](#common-quants)
|
||||
- [All Quants](#all-quants)
|
||||
- [Downloading using huggingface-cli](#downloading-using-huggingface-cli)
|
||||
- [Inference](#inference)
|
||||
- [Simple chat template](#simple-chat-template)
|
||||
- [Llama.cpp](#llama-cpp)
|
||||
- [FAQ](#faq)
|
||||
- [Why is the IMatrix not applied everywhere?](#why-is-the-imatrix-not-applied-everywhere)
|
||||
- [How do I merge a split GGUF?](#how-do-i-merge-a-split-gguf)
|
||||
|
||||
---
|
||||
|
||||
## Files
|
||||
|
||||
### IMatrix
|
||||
Status: ✅ Available
|
||||
Link: [here](https://huggingface.co/legraphista/NuminaMath-7B-TIR-IMat-GGUF/blob/main/imatrix.dat)
|
||||
|
||||
### Common Quants
|
||||
| Filename | Quant type | File Size | Status | Uses IMatrix | Is Split |
|
||||
| -------- | ---------- | --------- | ------ | ------------ | -------- |
|
||||
| [NuminaMath-7B-TIR.Q8_0.gguf](https://huggingface.co/legraphista/NuminaMath-7B-TIR-IMat-GGUF/blob/main/NuminaMath-7B-TIR.Q8_0.gguf) | Q8_0 | 7.35GB | ✅ Available | ⚪ Static | 📦 No
|
||||
| [NuminaMath-7B-TIR.Q6_K.gguf](https://huggingface.co/legraphista/NuminaMath-7B-TIR-IMat-GGUF/blob/main/NuminaMath-7B-TIR.Q6_K.gguf) | Q6_K | 5.67GB | ✅ Available | ⚪ Static | 📦 No
|
||||
| [NuminaMath-7B-TIR.Q4_K.gguf](https://huggingface.co/legraphista/NuminaMath-7B-TIR-IMat-GGUF/blob/main/NuminaMath-7B-TIR.Q4_K.gguf) | Q4_K | 4.22GB | ✅ Available | 🟢 IMatrix | 📦 No
|
||||
| [NuminaMath-7B-TIR.Q3_K.gguf](https://huggingface.co/legraphista/NuminaMath-7B-TIR-IMat-GGUF/blob/main/NuminaMath-7B-TIR.Q3_K.gguf) | Q3_K | 3.46GB | ✅ Available | 🟢 IMatrix | 📦 No
|
||||
| [NuminaMath-7B-TIR.Q2_K.gguf](https://huggingface.co/legraphista/NuminaMath-7B-TIR-IMat-GGUF/blob/main/NuminaMath-7B-TIR.Q2_K.gguf) | Q2_K | 2.72GB | ✅ Available | 🟢 IMatrix | 📦 No
|
||||
|
||||
|
||||
### All Quants
|
||||
| Filename | Quant type | File Size | Status | Uses IMatrix | Is Split |
|
||||
| -------- | ---------- | --------- | ------ | ------------ | -------- |
|
||||
| [NuminaMath-7B-TIR.BF16.gguf](https://huggingface.co/legraphista/NuminaMath-7B-TIR-IMat-GGUF/blob/main/NuminaMath-7B-TIR.BF16.gguf) | BF16 | 13.83GB | ✅ Available | ⚪ Static | 📦 No
|
||||
| [NuminaMath-7B-TIR.FP16.gguf](https://huggingface.co/legraphista/NuminaMath-7B-TIR-IMat-GGUF/blob/main/NuminaMath-7B-TIR.FP16.gguf) | F16 | 13.83GB | ✅ Available | ⚪ Static | 📦 No
|
||||
| [NuminaMath-7B-TIR.Q8_0.gguf](https://huggingface.co/legraphista/NuminaMath-7B-TIR-IMat-GGUF/blob/main/NuminaMath-7B-TIR.Q8_0.gguf) | Q8_0 | 7.35GB | ✅ Available | ⚪ Static | 📦 No
|
||||
| [NuminaMath-7B-TIR.Q6_K.gguf](https://huggingface.co/legraphista/NuminaMath-7B-TIR-IMat-GGUF/blob/main/NuminaMath-7B-TIR.Q6_K.gguf) | Q6_K | 5.67GB | ✅ Available | ⚪ Static | 📦 No
|
||||
| [NuminaMath-7B-TIR.Q5_K.gguf](https://huggingface.co/legraphista/NuminaMath-7B-TIR-IMat-GGUF/blob/main/NuminaMath-7B-TIR.Q5_K.gguf) | Q5_K | 4.93GB | ✅ Available | ⚪ Static | 📦 No
|
||||
| [NuminaMath-7B-TIR.Q5_K_S.gguf](https://huggingface.co/legraphista/NuminaMath-7B-TIR-IMat-GGUF/blob/main/NuminaMath-7B-TIR.Q5_K_S.gguf) | Q5_K_S | 4.81GB | ✅ Available | ⚪ Static | 📦 No
|
||||
| [NuminaMath-7B-TIR.Q4_K.gguf](https://huggingface.co/legraphista/NuminaMath-7B-TIR-IMat-GGUF/blob/main/NuminaMath-7B-TIR.Q4_K.gguf) | Q4_K | 4.22GB | ✅ Available | 🟢 IMatrix | 📦 No
|
||||
| [NuminaMath-7B-TIR.Q4_K_S.gguf](https://huggingface.co/legraphista/NuminaMath-7B-TIR-IMat-GGUF/blob/main/NuminaMath-7B-TIR.Q4_K_S.gguf) | Q4_K_S | 4.03GB | ✅ Available | 🟢 IMatrix | 📦 No
|
||||
| [NuminaMath-7B-TIR.IQ4_NL.gguf](https://huggingface.co/legraphista/NuminaMath-7B-TIR-IMat-GGUF/blob/main/NuminaMath-7B-TIR.IQ4_NL.gguf) | IQ4_NL | 4.00GB | ✅ Available | 🟢 IMatrix | 📦 No
|
||||
| [NuminaMath-7B-TIR.IQ4_XS.gguf](https://huggingface.co/legraphista/NuminaMath-7B-TIR-IMat-GGUF/blob/main/NuminaMath-7B-TIR.IQ4_XS.gguf) | IQ4_XS | 3.80GB | ✅ Available | 🟢 IMatrix | 📦 No
|
||||
| [NuminaMath-7B-TIR.Q3_K.gguf](https://huggingface.co/legraphista/NuminaMath-7B-TIR-IMat-GGUF/blob/main/NuminaMath-7B-TIR.Q3_K.gguf) | Q3_K | 3.46GB | ✅ Available | 🟢 IMatrix | 📦 No
|
||||
| [NuminaMath-7B-TIR.Q3_K_L.gguf](https://huggingface.co/legraphista/NuminaMath-7B-TIR-IMat-GGUF/blob/main/NuminaMath-7B-TIR.Q3_K_L.gguf) | Q3_K_L | 3.75GB | ✅ Available | 🟢 IMatrix | 📦 No
|
||||
| [NuminaMath-7B-TIR.Q3_K_S.gguf](https://huggingface.co/legraphista/NuminaMath-7B-TIR-IMat-GGUF/blob/main/NuminaMath-7B-TIR.Q3_K_S.gguf) | Q3_K_S | 3.14GB | ✅ Available | 🟢 IMatrix | 📦 No
|
||||
| [NuminaMath-7B-TIR.IQ3_M.gguf](https://huggingface.co/legraphista/NuminaMath-7B-TIR-IMat-GGUF/blob/main/NuminaMath-7B-TIR.IQ3_M.gguf) | IQ3_M | 3.29GB | ✅ Available | 🟢 IMatrix | 📦 No
|
||||
| [NuminaMath-7B-TIR.IQ3_S.gguf](https://huggingface.co/legraphista/NuminaMath-7B-TIR-IMat-GGUF/blob/main/NuminaMath-7B-TIR.IQ3_S.gguf) | IQ3_S | 3.14GB | ✅ Available | 🟢 IMatrix | 📦 No
|
||||
| [NuminaMath-7B-TIR.IQ3_XS.gguf](https://huggingface.co/legraphista/NuminaMath-7B-TIR-IMat-GGUF/blob/main/NuminaMath-7B-TIR.IQ3_XS.gguf) | IQ3_XS | 2.99GB | ✅ Available | 🟢 IMatrix | 📦 No
|
||||
| [NuminaMath-7B-TIR.IQ3_XXS.gguf](https://huggingface.co/legraphista/NuminaMath-7B-TIR-IMat-GGUF/blob/main/NuminaMath-7B-TIR.IQ3_XXS.gguf) | IQ3_XXS | 2.76GB | ✅ Available | 🟢 IMatrix | 📦 No
|
||||
| [NuminaMath-7B-TIR.Q2_K.gguf](https://huggingface.co/legraphista/NuminaMath-7B-TIR-IMat-GGUF/blob/main/NuminaMath-7B-TIR.Q2_K.gguf) | Q2_K | 2.72GB | ✅ Available | 🟢 IMatrix | 📦 No
|
||||
| [NuminaMath-7B-TIR.Q2_K_S.gguf](https://huggingface.co/legraphista/NuminaMath-7B-TIR-IMat-GGUF/blob/main/NuminaMath-7B-TIR.Q2_K_S.gguf) | Q2_K_S | 2.51GB | ✅ Available | 🟢 IMatrix | 📦 No
|
||||
| [NuminaMath-7B-TIR.IQ2_M.gguf](https://huggingface.co/legraphista/NuminaMath-7B-TIR-IMat-GGUF/blob/main/NuminaMath-7B-TIR.IQ2_M.gguf) | IQ2_M | 2.54GB | ✅ Available | 🟢 IMatrix | 📦 No
|
||||
| [NuminaMath-7B-TIR.IQ2_S.gguf](https://huggingface.co/legraphista/NuminaMath-7B-TIR-IMat-GGUF/blob/main/NuminaMath-7B-TIR.IQ2_S.gguf) | IQ2_S | 2.39GB | ✅ Available | 🟢 IMatrix | 📦 No
|
||||
| [NuminaMath-7B-TIR.IQ2_XS.gguf](https://huggingface.co/legraphista/NuminaMath-7B-TIR-IMat-GGUF/blob/main/NuminaMath-7B-TIR.IQ2_XS.gguf) | IQ2_XS | 2.21GB | ✅ Available | 🟢 IMatrix | 📦 No
|
||||
| [NuminaMath-7B-TIR.IQ2_XXS.gguf](https://huggingface.co/legraphista/NuminaMath-7B-TIR-IMat-GGUF/blob/main/NuminaMath-7B-TIR.IQ2_XXS.gguf) | IQ2_XXS | 2.04GB | ✅ Available | 🟢 IMatrix | 📦 No
|
||||
| [NuminaMath-7B-TIR.IQ1_M.gguf](https://huggingface.co/legraphista/NuminaMath-7B-TIR-IMat-GGUF/blob/main/NuminaMath-7B-TIR.IQ1_M.gguf) | IQ1_M | 1.85GB | ✅ Available | 🟢 IMatrix | 📦 No
|
||||
| [NuminaMath-7B-TIR.IQ1_S.gguf](https://huggingface.co/legraphista/NuminaMath-7B-TIR-IMat-GGUF/blob/main/NuminaMath-7B-TIR.IQ1_S.gguf) | IQ1_S | 1.73GB | ✅ Available | 🟢 IMatrix | 📦 No
|
||||
|
||||
|
||||
## Downloading using huggingface-cli
|
||||
If you do not have hugginface-cli installed:
|
||||
```
|
||||
pip install -U "huggingface_hub[cli]"
|
||||
```
|
||||
Download the specific file you want:
|
||||
```
|
||||
huggingface-cli download legraphista/NuminaMath-7B-TIR-IMat-GGUF --include "NuminaMath-7B-TIR.Q8_0.gguf" --local-dir ./
|
||||
```
|
||||
If the model file is big, it has been split into multiple files. In order to download them all to a local folder, run:
|
||||
```
|
||||
huggingface-cli download legraphista/NuminaMath-7B-TIR-IMat-GGUF --include "NuminaMath-7B-TIR.Q8_0/*" --local-dir ./
|
||||
# see FAQ for merging GGUF's
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Inference
|
||||
|
||||
### Simple chat template
|
||||
```
|
||||
### Problem: {user_prompt}
|
||||
### Solution: {assistant_response}
|
||||
### Problem: {next_user_prompt}
|
||||
|
||||
```
|
||||
|
||||
### Llama.cpp
|
||||
```
|
||||
llama.cpp/main -m NuminaMath-7B-TIR.Q8_0.gguf --color -i -p "prompt here (according to the chat template)"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## FAQ
|
||||
|
||||
### Why is the IMatrix not applied everywhere?
|
||||
According to [this investigation](https://www.reddit.com/r/LocalLLaMA/comments/1993iro/ggufs_quants_can_punch_above_their_weights_now/), it appears that lower quantizations are the only ones that benefit from the imatrix input (as per hellaswag results).
|
||||
|
||||
### How do I merge a split GGUF?
|
||||
1. Make sure you have `gguf-split` available
|
||||
- To get hold of `gguf-split`, navigate to https://github.com/ggerganov/llama.cpp/releases
|
||||
- Download the appropriate zip for your system from the latest release
|
||||
- Unzip the archive and you should be able to find `gguf-split`
|
||||
2. Locate your GGUF chunks folder (ex: `NuminaMath-7B-TIR.Q8_0`)
|
||||
3. Run `gguf-split --merge NuminaMath-7B-TIR.Q8_0/NuminaMath-7B-TIR.Q8_0-00001-of-XXXXX.gguf NuminaMath-7B-TIR.Q8_0.gguf`
|
||||
- Make sure to point `gguf-split` to the first chunk of the split.
|
||||
|
||||
---
|
||||
|
||||
Got a suggestion? Ping me [@legraphista](https://x.com/legraphista)!
|
||||
3
imatrix.dat
Normal file
3
imatrix.dat
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:006f8474518157f71af16f069b53a18794b0fee85f31f0799caa3f6817634491
|
||||
size 4277055
|
||||
2482
imatrix.dataset
Normal file
2482
imatrix.dataset
Normal file
File diff suppressed because one or more lines are too long
145
imatrix.log
Normal file
145
imatrix.log
Normal file
@@ -0,0 +1,145 @@
|
||||
llama_model_loader: loaded meta data with 25 key-value pairs and 273 tensors from NuminaMath-7B-TIR-IMat-GGUF/NuminaMath-7B-TIR.Q8_0.gguf.hardlink.gguf (version GGUF V3 (latest))
|
||||
llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.
|
||||
llama_model_loader: - kv 0: general.architecture str = llama
|
||||
llama_model_loader: - kv 1: general.name str = NuminaMath-7B-TIR
|
||||
llama_model_loader: - kv 2: llama.block_count u32 = 30
|
||||
llama_model_loader: - kv 3: llama.context_length u32 = 4096
|
||||
llama_model_loader: - kv 4: llama.embedding_length u32 = 4096
|
||||
llama_model_loader: - kv 5: llama.feed_forward_length u32 = 11008
|
||||
llama_model_loader: - kv 6: llama.attention.head_count u32 = 32
|
||||
llama_model_loader: - kv 7: llama.attention.head_count_kv u32 = 32
|
||||
llama_model_loader: - kv 8: llama.rope.freq_base f32 = 10000.000000
|
||||
llama_model_loader: - kv 9: llama.attention.layer_norm_rms_epsilon f32 = 0.000001
|
||||
llama_model_loader: - kv 10: general.file_type u32 = 7
|
||||
llama_model_loader: - kv 11: llama.vocab_size u32 = 102400
|
||||
llama_model_loader: - kv 12: llama.rope.dimension_count u32 = 128
|
||||
llama_model_loader: - kv 13: tokenizer.ggml.model str = gpt2
|
||||
llama_model_loader: - kv 14: tokenizer.ggml.pre str = deepseek-llm
|
||||
llama_model_loader: - kv 15: tokenizer.ggml.tokens arr[str,102400] = ["!", "\"", "#", "$", "%", "&", "'", ...
|
||||
llama_model_loader: - kv 16: tokenizer.ggml.token_type arr[i32,102400] = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...
|
||||
llama_model_loader: - kv 17: tokenizer.ggml.merges arr[str,99757] = ["Ġ Ġ", "Ġ t", "Ġ a", "i n", "h e...
|
||||
llama_model_loader: - kv 18: tokenizer.ggml.bos_token_id u32 = 100000
|
||||
llama_model_loader: - kv 19: tokenizer.ggml.eos_token_id u32 = 100001
|
||||
llama_model_loader: - kv 20: tokenizer.ggml.padding_token_id u32 = 100001
|
||||
llama_model_loader: - kv 21: tokenizer.ggml.add_bos_token bool = true
|
||||
llama_model_loader: - kv 22: tokenizer.ggml.add_eos_token bool = false
|
||||
llama_model_loader: - kv 23: tokenizer.chat_template str = {% for message in messages %}{% if (m...
|
||||
llama_model_loader: - kv 24: general.quantization_version u32 = 2
|
||||
llama_model_loader: - type f32: 61 tensors
|
||||
llama_model_loader: - type q8_0: 212 tensors
|
||||
llm_load_vocab: special tokens cache size = 2400
|
||||
llm_load_vocab: token to piece cache size = 0.6659 MB
|
||||
llm_load_print_meta: format = GGUF V3 (latest)
|
||||
llm_load_print_meta: arch = llama
|
||||
llm_load_print_meta: vocab type = BPE
|
||||
llm_load_print_meta: n_vocab = 102400
|
||||
llm_load_print_meta: n_merges = 99757
|
||||
llm_load_print_meta: vocab_only = 0
|
||||
llm_load_print_meta: n_ctx_train = 4096
|
||||
llm_load_print_meta: n_embd = 4096
|
||||
llm_load_print_meta: n_layer = 30
|
||||
llm_load_print_meta: n_head = 32
|
||||
llm_load_print_meta: n_head_kv = 32
|
||||
llm_load_print_meta: n_rot = 128
|
||||
llm_load_print_meta: n_swa = 0
|
||||
llm_load_print_meta: n_embd_head_k = 128
|
||||
llm_load_print_meta: n_embd_head_v = 128
|
||||
llm_load_print_meta: n_gqa = 1
|
||||
llm_load_print_meta: n_embd_k_gqa = 4096
|
||||
llm_load_print_meta: n_embd_v_gqa = 4096
|
||||
llm_load_print_meta: f_norm_eps = 0.0e+00
|
||||
llm_load_print_meta: f_norm_rms_eps = 1.0e-06
|
||||
llm_load_print_meta: f_clamp_kqv = 0.0e+00
|
||||
llm_load_print_meta: f_max_alibi_bias = 0.0e+00
|
||||
llm_load_print_meta: f_logit_scale = 0.0e+00
|
||||
llm_load_print_meta: n_ff = 11008
|
||||
llm_load_print_meta: n_expert = 0
|
||||
llm_load_print_meta: n_expert_used = 0
|
||||
llm_load_print_meta: causal attn = 1
|
||||
llm_load_print_meta: pooling type = 0
|
||||
llm_load_print_meta: rope type = 0
|
||||
llm_load_print_meta: rope scaling = linear
|
||||
llm_load_print_meta: freq_base_train = 10000.0
|
||||
llm_load_print_meta: freq_scale_train = 1
|
||||
llm_load_print_meta: n_ctx_orig_yarn = 4096
|
||||
llm_load_print_meta: rope_finetuned = unknown
|
||||
llm_load_print_meta: ssm_d_conv = 0
|
||||
llm_load_print_meta: ssm_d_inner = 0
|
||||
llm_load_print_meta: ssm_d_state = 0
|
||||
llm_load_print_meta: ssm_dt_rank = 0
|
||||
llm_load_print_meta: model type = ?B
|
||||
llm_load_print_meta: model ftype = Q8_0
|
||||
llm_load_print_meta: model params = 6.91 B
|
||||
llm_load_print_meta: model size = 6.84 GiB (8.50 BPW)
|
||||
llm_load_print_meta: general.name = NuminaMath-7B-TIR
|
||||
llm_load_print_meta: BOS token = 100000 '<｜begin▁of▁sentence｜>'
|
||||
llm_load_print_meta: EOS token = 100001 '<｜end▁of▁sentence｜>'
|
||||
llm_load_print_meta: PAD token = 100001 '<｜end▁of▁sentence｜>'
|
||||
llm_load_print_meta: LF token = 126 'Ä'
|
||||
llm_load_print_meta: max token length = 256
|
||||
ggml_cuda_init: GGML_CUDA_FORCE_MMQ: no
|
||||
ggml_cuda_init: GGML_CUDA_FORCE_CUBLAS: no
|
||||
ggml_cuda_init: found 1 CUDA devices:
|
||||
Device 0: NVIDIA GeForce RTX 4090, compute capability 8.9, VMM: yes
|
||||
llm_load_tensors: ggml ctx size = 0.26 MiB
|
||||
llm_load_tensors: offloading 30 repeating layers to GPU
|
||||
llm_load_tensors: offloading non-repeating layers to GPU
|
||||
llm_load_tensors: offloaded 31/31 layers to GPU
|
||||
llm_load_tensors: CPU buffer size = 425.00 MiB
|
||||
llm_load_tensors: CUDA0 buffer size = 6577.84 MiB
|
||||
..........................................................................................
|
||||
llama_new_context_with_model: n_ctx = 512
|
||||
llama_new_context_with_model: n_batch = 512
|
||||
llama_new_context_with_model: n_ubatch = 512
|
||||
llama_new_context_with_model: flash_attn = 0
|
||||
llama_new_context_with_model: freq_base = 10000.0
|
||||
llama_new_context_with_model: freq_scale = 1
|
||||
llama_kv_cache_init: CUDA0 KV buffer size = 240.00 MiB
|
||||
llama_new_context_with_model: KV self size = 240.00 MiB, K (f16): 120.00 MiB, V (f16): 120.00 MiB
|
||||
llama_new_context_with_model: CUDA_Host output buffer size = 0.39 MiB
|
||||
llama_new_context_with_model: CUDA0 compute buffer size = 208.00 MiB
|
||||
llama_new_context_with_model: CUDA_Host compute buffer size = 9.01 MiB
|
||||
llama_new_context_with_model: graph nodes = 966
|
||||
llama_new_context_with_model: graph splits = 2
|
||||
|
||||
system_info: n_threads = 25 / 32 | AVX = 1 | AVX_VNNI = 0 | AVX2 = 1 | AVX512 = 1 | AVX512_VBMI = 1 | AVX512_VNNI = 1 | AVX512_BF16 = 1 | FMA = 1 | NEON = 0 | SVE = 0 | ARM_FMA = 0 | F16C = 1 | FP16_VA = 0 | WASM_SIMD = 0 | BLAS = 1 | SSE3 = 1 | SSSE3 = 1 | VSX = 0 | MATMUL_INT8 = 0 | LLAMAFILE = 1 |
|
||||
compute_imatrix: tokenizing the input ..
|
||||
compute_imatrix: tokenization took 219.795 ms
|
||||
compute_imatrix: computing over 139 chunks with batch_size 512
|
||||
compute_imatrix: 0.66 seconds per pass - ETA 1.53 minutes
|
||||
[1]10.4314,[2]7.3682,[3]6.7831,[4]8.2509,[5]7.7539,[6]7.2769,[7]8.2499,[8]8.2793,[9]9.5899,
|
||||
save_imatrix: stored collected data after 10 chunks in NuminaMath-7B-TIR-IMat-GGUF/imatrix.dat
|
||||
[10]9.8051,[11]9.0158,[12]9.8822,[13]10.7192,[14]11.4750,[15]11.7074,[16]12.4342,[17]12.8366,[18]13.0696,[19]13.6454,
|
||||
save_imatrix: stored collected data after 20 chunks in NuminaMath-7B-TIR-IMat-GGUF/imatrix.dat
|
||||
[20]12.7369,[21]12.7164,[22]13.0572,[23]13.4045,[24]13.0591,[25]13.4993,[26]13.0929,[27]13.5825,[28]13.4809,[29]13.9218,
|
||||
save_imatrix: stored collected data after 30 chunks in NuminaMath-7B-TIR-IMat-GGUF/imatrix.dat
|
||||
[30]14.3508,[31]14.8240,[32]14.6801,[33]14.0268,[34]13.0593,[35]12.3150,[36]12.1439,[37]12.1304,[38]12.0899,[39]11.8205,
|
||||
save_imatrix: stored collected data after 40 chunks in NuminaMath-7B-TIR-IMat-GGUF/imatrix.dat
|
||||
[40]11.7968,[41]11.4809,[42]11.2521,[43]11.4326,[44]11.5349,[45]11.8091,[46]11.8210,[47]12.4688,[48]12.9043,[49]13.2747,
|
||||
save_imatrix: stored collected data after 50 chunks in NuminaMath-7B-TIR-IMat-GGUF/imatrix.dat
|
||||
[50]13.5610,[51]13.7512,[52]13.5889,[53]13.7792,[54]14.0260,[55]14.1128,[56]13.9351,[57]13.8070,[58]13.7664,[59]13.9588,
|
||||
save_imatrix: stored collected data after 60 chunks in NuminaMath-7B-TIR-IMat-GGUF/imatrix.dat
|
||||
[60]14.1955,[61]14.4787,[62]14.5778,[63]14.6177,[64]14.6769,[65]14.6685,[66]14.6656,[67]14.6345,[68]14.5578,[69]14.6843,
|
||||
save_imatrix: stored collected data after 70 chunks in NuminaMath-7B-TIR-IMat-GGUF/imatrix.dat
|
||||
[70]14.8954,[71]14.8467,[72]14.8260,[73]14.7263,[74]14.6219,[75]14.4769,[76]14.3974,[77]14.3304,[78]14.2765,[79]14.1135,
|
||||
save_imatrix: stored collected data after 80 chunks in NuminaMath-7B-TIR-IMat-GGUF/imatrix.dat
|
||||
[80]14.0793,[81]14.0205,[82]13.9517,[83]13.8239,[84]13.7387,[85]13.6672,[86]13.5425,[87]13.4674,[88]13.4413,[89]13.4553,
|
||||
save_imatrix: stored collected data after 90 chunks in NuminaMath-7B-TIR-IMat-GGUF/imatrix.dat
|
||||
[90]13.3877,[91]13.4239,[92]13.4322,[93]13.3130,[94]13.2679,[95]13.2266,[96]13.3477,[97]13.4149,[98]13.4163,[99]13.2541,
|
||||
save_imatrix: stored collected data after 100 chunks in NuminaMath-7B-TIR-IMat-GGUF/imatrix.dat
|
||||
[100]13.0952,[101]12.9394,[102]12.7769,[103]12.6021,[104]12.4604,[105]12.3279,[106]12.1804,[107]12.0326,[108]11.9975,[109]12.0266,
|
||||
save_imatrix: stored collected data after 110 chunks in NuminaMath-7B-TIR-IMat-GGUF/imatrix.dat
|
||||
[110]12.0979,[111]12.2131,[112]12.3274,[113]12.4241,[114]12.6192,[115]12.7171,[116]12.7818,[117]12.7623,[118]12.8884,[119]12.8693,
|
||||
save_imatrix: stored collected data after 120 chunks in NuminaMath-7B-TIR-IMat-GGUF/imatrix.dat
|
||||
[120]12.8420,[121]12.7717,[122]12.7185,[123]12.8016,[124]12.8760,[125]12.8461,[126]12.8485,[127]12.8573,[128]12.9072,[129]12.9172,
|
||||
save_imatrix: stored collected data after 130 chunks in NuminaMath-7B-TIR-IMat-GGUF/imatrix.dat
|
||||
[130]12.9305,[131]12.9718,[132]12.9349,[133]12.8673,[134]12.9903,[135]13.1251,[136]13.2243,[137]13.3934,[138]13.5698,[139]13.6705,
|
||||
save_imatrix: stored collected data after 139 chunks in NuminaMath-7B-TIR-IMat-GGUF/imatrix.dat
|
||||
|
||||
llama_print_timings: load time = 9195.41 ms
|
||||
llama_print_timings: sample time = 0.00 ms / 1 runs ( 0.00 ms per token, inf tokens per second)
|
||||
llama_print_timings: prompt eval time = 71105.13 ms / 71168 tokens ( 1.00 ms per token, 1000.88 tokens per second)
|
||||
llama_print_timings: eval time = 0.00 ms / 1 runs ( 0.00 ms per token, inf tokens per second)
|
||||
llama_print_timings: total time = 80417.25 ms / 71169 tokens
|
||||
|
||||
Final estimate: PPL = 13.6705 +/- 0.25616
|
||||
Reference in New Issue
Block a user