初始化项目，由ModelHub XC社区提供模型
Model: legraphista/Phi-3-medium-128k-instruct-IMat-GGUF Source: Original Platform
This commit is contained in:
ModelHub XC
61
.gitattributes
vendored
Normal file
61
.gitattributes
vendored
Normal file
@@ -0,0 +1,61 @@
|
||||
*.7z filter=lfs diff=lfs merge=lfs -text
|
||||
*.arrow filter=lfs diff=lfs merge=lfs -text
|
||||
*.bin filter=lfs diff=lfs merge=lfs -text
|
||||
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
||||
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
||||
*.ftz filter=lfs diff=lfs merge=lfs -text
|
||||
*.gz filter=lfs diff=lfs merge=lfs -text
|
||||
*.h5 filter=lfs diff=lfs merge=lfs -text
|
||||
*.joblib filter=lfs diff=lfs merge=lfs -text
|
||||
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
||||
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
||||
*.model filter=lfs diff=lfs merge=lfs -text
|
||||
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
||||
*.npy filter=lfs diff=lfs merge=lfs -text
|
||||
*.npz filter=lfs diff=lfs merge=lfs -text
|
||||
*.onnx filter=lfs diff=lfs merge=lfs -text
|
||||
*.ot filter=lfs diff=lfs merge=lfs -text
|
||||
*.parquet filter=lfs diff=lfs merge=lfs -text
|
||||
*.pb filter=lfs diff=lfs merge=lfs -text
|
||||
*.pickle filter=lfs diff=lfs merge=lfs -text
|
||||
*.pkl filter=lfs diff=lfs merge=lfs -text
|
||||
*.pt filter=lfs diff=lfs merge=lfs -text
|
||||
*.pth filter=lfs diff=lfs merge=lfs -text
|
||||
*.rar filter=lfs diff=lfs merge=lfs -text
|
||||
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
||||
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
||||
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
||||
*.tar filter=lfs diff=lfs merge=lfs -text
|
||||
*.tflite filter=lfs diff=lfs merge=lfs -text
|
||||
*.tgz filter=lfs diff=lfs merge=lfs -text
|
||||
*.wasm filter=lfs diff=lfs merge=lfs -text
|
||||
*.xz filter=lfs diff=lfs merge=lfs -text
|
||||
*.zip filter=lfs diff=lfs merge=lfs -text
|
||||
*.zst filter=lfs diff=lfs merge=lfs -text
|
||||
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
||||
Phi-3-medium-128k-instruct.Q6_K.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
Phi-3-medium-128k-instruct.Q8_0.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
Phi-3-medium-128k-instruct.Q5_K.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
Phi-3-medium-128k-instruct.FP16.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
Phi-3-medium-128k-instruct.Q5_K_S.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
Phi-3-medium-128k-instruct.BF16.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
imatrix.dat filter=lfs diff=lfs merge=lfs -text
|
||||
Phi-3-medium-128k-instruct.Q4_K.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
Phi-3-medium-128k-instruct.Q2_K.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
Phi-3-medium-128k-instruct.Q3_K.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
Phi-3-medium-128k-instruct.Q3_K_S.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
Phi-3-medium-128k-instruct.Q3_K_L.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
Phi-3-medium-128k-instruct.Q4_K_S.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
Phi-3-medium-128k-instruct.Q2_K_S.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
Phi-3-medium-128k-instruct.IQ4_NL.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
Phi-3-medium-128k-instruct.IQ4_XS.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
Phi-3-medium-128k-instruct.IQ3_M.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
Phi-3-medium-128k-instruct.IQ3_S.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
Phi-3-medium-128k-instruct.IQ3_XS.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
Phi-3-medium-128k-instruct.IQ3_XXS.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
Phi-3-medium-128k-instruct.IQ2_M.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
Phi-3-medium-128k-instruct.IQ2_S.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
Phi-3-medium-128k-instruct.IQ2_XS.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
Phi-3-medium-128k-instruct.IQ2_XXS.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
Phi-3-medium-128k-instruct.IQ1_M.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
Phi-3-medium-128k-instruct.IQ1_S.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
3
Phi-3-medium-128k-instruct.BF16.gguf
Normal file
3
Phi-3-medium-128k-instruct.BF16.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:c82493237cbb1b4f3130e9611c6e0221c8e6d3799dd8d8c1ae9b9f2e9d8bc98d
|
||||
size 27922046720
|
||||
3
Phi-3-medium-128k-instruct.FP16.gguf
Normal file
3
Phi-3-medium-128k-instruct.FP16.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:9d44b5302a369a2c7da61f6bc1850be70f7777d70a3446afde0fd91ffc8c7a02
|
||||
size 27922046720
|
||||
3
Phi-3-medium-128k-instruct.IQ1_M.gguf
Normal file
3
Phi-3-medium-128k-instruct.IQ1_M.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:20999a2ed806713242511d1a96d1702bd0cea31b07938a34be51d51e8104348a
|
||||
size 3242157056
|
||||
3
Phi-3-medium-128k-instruct.IQ1_S.gguf
Normal file
3
Phi-3-medium-128k-instruct.IQ1_S.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:eec7e84908608a8981baa56f20a480e4def20773da149e403caa81c25ac0d607
|
||||
size 2957997056
|
||||
3
Phi-3-medium-128k-instruct.IQ2_M.gguf
Normal file
3
Phi-3-medium-128k-instruct.IQ2_M.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:c83a68bdd1c9b1aa89fad423b8800d9ec309d46e7b128eeb3b4e1e58bcd81224
|
||||
size 4717006336
|
||||
3
Phi-3-medium-128k-instruct.IQ2_S.gguf
Normal file
3
Phi-3-medium-128k-instruct.IQ2_S.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:fc6ceada673efaed6d2672c5a992478799063773aa7b516916978f22a18656d4
|
||||
size 4338126336
|
||||
3
Phi-3-medium-128k-instruct.IQ2_XS.gguf
Normal file
3
Phi-3-medium-128k-instruct.IQ2_XS.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:00af4a54c43c4fce7f362adc76e8d8f7879c8a61365a83e3da1790607e12ee5f
|
||||
size 4127405056
|
||||
3
Phi-3-medium-128k-instruct.IQ2_XXS.gguf
Normal file
3
Phi-3-medium-128k-instruct.IQ2_XXS.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:516ed52a141b2cc9a9fd2d5c22cb4c97eb8246a723b397c88f54e0c500555a29
|
||||
size 3715757056
|
||||
3
Phi-3-medium-128k-instruct.IQ3_M.gguf
Normal file
3
Phi-3-medium-128k-instruct.IQ3_M.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:27cc1f70b388ad6bc1758087d02d408f1c74c9e34d0dfd8bfd2971a7310ff28e
|
||||
size 6473977856
|
||||
3
Phi-3-medium-128k-instruct.IQ3_S.gguf
Normal file
3
Phi-3-medium-128k-instruct.IQ3_S.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:1c0c40e462e9c82e86e88339a30089961cf4c6a1793a0b3643891ebc761e634a
|
||||
size 6064889856
|
||||
3
Phi-3-medium-128k-instruct.IQ3_XS.gguf
Normal file
3
Phi-3-medium-128k-instruct.IQ3_XS.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:19e6932d9b3b9fa3949d2a09b2b1b09346abe3425e20ef93e2d05d21527b1563
|
||||
size 5806841856
|
||||
3
Phi-3-medium-128k-instruct.IQ3_XXS.gguf
Normal file
3
Phi-3-medium-128k-instruct.IQ3_XXS.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:1fa70397d109c90d4a465431e8e17ba07ad641b1841a8ca9239f1bd3cd71c1de
|
||||
size 5453262336
|
||||
3
Phi-3-medium-128k-instruct.IQ4_NL.gguf
Normal file
3
Phi-3-medium-128k-instruct.IQ4_NL.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:d41ead77c39f81e7bf28063e02e958e6ba75201889b3220267462bd6cd961599
|
||||
size 7897125376
|
||||
3
Phi-3-medium-128k-instruct.IQ4_XS.gguf
Normal file
3
Phi-3-medium-128k-instruct.IQ4_XS.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:e1cc51b03d4cb1edb8b4e14f93b2678719dda169408f91a7edf63d39c6cd2b4b
|
||||
size 7466011136
|
||||
3
Phi-3-medium-128k-instruct.Q2_K.gguf
Normal file
3
Phi-3-medium-128k-instruct.Q2_K.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:21881a1476e0369c15961b29f6fb8b517c3b94d277033d5ac3e74f0e18a67f05
|
||||
size 5143000576
|
||||
3
Phi-3-medium-128k-instruct.Q2_K_S.gguf
Normal file
3
Phi-3-medium-128k-instruct.Q2_K_S.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:ca1df9c891ee635732d60fff80bc79adede44938dafc699383061f47d75efd68
|
||||
size 4771288576
|
||||
3
Phi-3-medium-128k-instruct.Q3_K.gguf
Normal file
3
Phi-3-medium-128k-instruct.Q3_K.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:ca1dc171e94f23d6730e9a215612a3be7a148b46cdd1363cfa599f567f40ad7d
|
||||
size 6923411456
|
||||
3
Phi-3-medium-128k-instruct.Q3_K_L.gguf
Normal file
3
Phi-3-medium-128k-instruct.Q3_K_L.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:65f7f0a5831e08d5bfcb3a641f53eda0d760f3fbbee15e8845105d4fbaf78a48
|
||||
size 7490297856
|
||||
3
Phi-3-medium-128k-instruct.Q3_K_S.gguf
Normal file
3
Phi-3-medium-128k-instruct.Q3_K_S.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:c67fa726b5cdff08bb2db8070b3f31e49cd2900851b29de7c7c9b419146b8ea3
|
||||
size 6064889856
|
||||
3
Phi-3-medium-128k-instruct.Q4_K.gguf
Normal file
3
Phi-3-medium-128k-instruct.Q4_K.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:6514db096a048731f963fdc57e6a55cfc64eeb8e1ac82abb00798f9fc9825982
|
||||
size 8566821376
|
||||
3
Phi-3-medium-128k-instruct.Q4_K_S.gguf
Normal file
3
Phi-3-medium-128k-instruct.Q4_K_S.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:31bb7fc7112c59c4e868375c056d30bece2b0a6f5b8dbfa429d633ed7cfd11d6
|
||||
size 7954469376
|
||||
3
Phi-3-medium-128k-instruct.Q5_K.gguf
Normal file
3
Phi-3-medium-128k-instruct.Q5_K.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:fa9f5542bbeec685ed624f5cf13c64805e2f58771fa5abdb5e59fddfac3760e0
|
||||
size 10074190080
|
||||
3
Phi-3-medium-128k-instruct.Q5_K_S.gguf
Normal file
3
Phi-3-medium-128k-instruct.Q5_K_S.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:ac6c587de96419673833b1cb3d718fd72c4b84bf35f894785c74273c7c3acc55
|
||||
size 9621582080
|
||||
3
Phi-3-medium-128k-instruct.Q6_K.gguf
Normal file
3
Phi-3-medium-128k-instruct.Q6_K.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:0b65eb05a6c61a7c1b87bb720d33067f53feb793e7fca081786b5b4e1f73e06c
|
||||
size 11453817600
|
||||
3
Phi-3-medium-128k-instruct.Q8_0.gguf
Normal file
3
Phi-3-medium-128k-instruct.Q8_0.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:66018dde2bc97cb092de6eb9804ac7cca6b665e91ad9a6e2fad808dcc75add11
|
||||
size 14834712320
|
||||
141
README.md
Normal file
141
README.md
Normal file
@@ -0,0 +1,141 @@
|
||||
---
|
||||
base_model: microsoft/Phi-3-medium-128k-instruct
|
||||
inference: false
|
||||
language:
|
||||
- multilingual
|
||||
library_name: gguf
|
||||
license: mit
|
||||
license_link: https://huggingface.co/microsoft/Phi-3-medium-128k-instruct/resolve/main/LICENSE
|
||||
pipeline_tag: text-generation
|
||||
quantized_by: legraphista
|
||||
tags:
|
||||
- quantized
|
||||
- GGUF
|
||||
- imatrix
|
||||
- quantization
|
||||
- imat
|
||||
- imatrix
|
||||
- static
|
||||
---
|
||||
|
||||
# Phi-3-medium-128k-instruct-IMat-GGUF
|
||||
_Llama.cpp imatrix quantization of microsoft/Phi-3-medium-128k-instruct_
|
||||
|
||||
Original Model: [microsoft/Phi-3-medium-128k-instruct](https://huggingface.co/microsoft/Phi-3-medium-128k-instruct)
|
||||
Original dtype: `BF16` (`bfloat16`)
|
||||
Quantized by: llama.cpp [b3003](https://github.com/ggerganov/llama.cpp/releases/tag/b3003)
|
||||
IMatrix dataset: [here](https://gist.githubusercontent.com/legraphista/d6d93f1a254bcfc58e0af3777eaec41e/raw/d380e7002cea4a51c33fffd47db851942754e7cc/imatrix.calibration.medium.raw)
|
||||
|
||||
- [Phi-3-medium-128k-instruct-IMat-GGUF](#phi-3-medium-128k-instruct-imat-gguf)
|
||||
- [Files](#files)
|
||||
- [IMatrix](#imatrix)
|
||||
- [Common Quants](#common-quants)
|
||||
- [All Quants](#all-quants)
|
||||
- [Downloading using huggingface-cli](#downloading-using-huggingface-cli)
|
||||
- [Inference](#inference)
|
||||
- [Simple chat template](#simple-chat-template)
|
||||
- [Llama.cpp](#llama-cpp)
|
||||
- [FAQ](#faq)
|
||||
- [Why is the IMatrix not applied everywhere?](#why-is-the-imatrix-not-applied-everywhere)
|
||||
- [How do I merge a split GGUF?](#how-do-i-merge-a-split-gguf)
|
||||
|
||||
---
|
||||
|
||||
## Files
|
||||
|
||||
### IMatrix
|
||||
Status: ✅ Available
|
||||
Link: [here](https://huggingface.co/legraphista/Phi-3-medium-128k-instruct-IMat-GGUF/blob/main/imatrix.dat)
|
||||
|
||||
### Common Quants
|
||||
| Filename | Quant type | File Size | Status | Uses IMatrix | Is Split |
|
||||
| -------- | ---------- | --------- | ------ | ------------ | -------- |
|
||||
| [Phi-3-medium-128k-instruct.Q8_0.gguf](https://huggingface.co/legraphista/Phi-3-medium-128k-instruct-IMat-GGUF/blob/main/Phi-3-medium-128k-instruct.Q8_0.gguf) | Q8_0 | 14.83GB | ✅ Available | ⚪ No | 📦 No
|
||||
| [Phi-3-medium-128k-instruct.Q6_K.gguf](https://huggingface.co/legraphista/Phi-3-medium-128k-instruct-IMat-GGUF/blob/main/Phi-3-medium-128k-instruct.Q6_K.gguf) | Q6_K | 11.45GB | ✅ Available | ⚪ No | 📦 No
|
||||
| [Phi-3-medium-128k-instruct.Q4_K.gguf](https://huggingface.co/legraphista/Phi-3-medium-128k-instruct-IMat-GGUF/blob/main/Phi-3-medium-128k-instruct.Q4_K.gguf) | Q4_K | 8.57GB | ✅ Available | 🟢 Yes | 📦 No
|
||||
| [Phi-3-medium-128k-instruct.Q3_K.gguf](https://huggingface.co/legraphista/Phi-3-medium-128k-instruct-IMat-GGUF/blob/main/Phi-3-medium-128k-instruct.Q3_K.gguf) | Q3_K | 6.92GB | ✅ Available | 🟢 Yes | 📦 No
|
||||
| [Phi-3-medium-128k-instruct.Q2_K.gguf](https://huggingface.co/legraphista/Phi-3-medium-128k-instruct-IMat-GGUF/blob/main/Phi-3-medium-128k-instruct.Q2_K.gguf) | Q2_K | 5.14GB | ✅ Available | 🟢 Yes | 📦 No
|
||||
|
||||
|
||||
### All Quants
|
||||
| Filename | Quant type | File Size | Status | Uses IMatrix | Is Split |
|
||||
| -------- | ---------- | --------- | ------ | ------------ | -------- |
|
||||
| [Phi-3-medium-128k-instruct.FP16.gguf](https://huggingface.co/legraphista/Phi-3-medium-128k-instruct-IMat-GGUF/blob/main/Phi-3-medium-128k-instruct.FP16.gguf) | F16 | 27.92GB | ✅ Available | ⚪ No | 📦 No
|
||||
| [Phi-3-medium-128k-instruct.BF16.gguf](https://huggingface.co/legraphista/Phi-3-medium-128k-instruct-IMat-GGUF/blob/main/Phi-3-medium-128k-instruct.BF16.gguf) | BF16 | 27.92GB | ✅ Available | ⚪ No | 📦 No
|
||||
| [Phi-3-medium-128k-instruct.Q5_K.gguf](https://huggingface.co/legraphista/Phi-3-medium-128k-instruct-IMat-GGUF/blob/main/Phi-3-medium-128k-instruct.Q5_K.gguf) | Q5_K | 10.07GB | ✅ Available | ⚪ No | 📦 No
|
||||
| [Phi-3-medium-128k-instruct.Q5_K_S.gguf](https://huggingface.co/legraphista/Phi-3-medium-128k-instruct-IMat-GGUF/blob/main/Phi-3-medium-128k-instruct.Q5_K_S.gguf) | Q5_K_S | 9.62GB | ✅ Available | ⚪ No | 📦 No
|
||||
| [Phi-3-medium-128k-instruct.Q4_K_S.gguf](https://huggingface.co/legraphista/Phi-3-medium-128k-instruct-IMat-GGUF/blob/main/Phi-3-medium-128k-instruct.Q4_K_S.gguf) | Q4_K_S | 7.95GB | ✅ Available | 🟢 Yes | 📦 No
|
||||
| [Phi-3-medium-128k-instruct.Q3_K_L.gguf](https://huggingface.co/legraphista/Phi-3-medium-128k-instruct-IMat-GGUF/blob/main/Phi-3-medium-128k-instruct.Q3_K_L.gguf) | Q3_K_L | 7.49GB | ✅ Available | 🟢 Yes | 📦 No
|
||||
| [Phi-3-medium-128k-instruct.Q3_K_S.gguf](https://huggingface.co/legraphista/Phi-3-medium-128k-instruct-IMat-GGUF/blob/main/Phi-3-medium-128k-instruct.Q3_K_S.gguf) | Q3_K_S | 6.06GB | ✅ Available | 🟢 Yes | 📦 No
|
||||
| [Phi-3-medium-128k-instruct.Q2_K_S.gguf](https://huggingface.co/legraphista/Phi-3-medium-128k-instruct-IMat-GGUF/blob/main/Phi-3-medium-128k-instruct.Q2_K_S.gguf) | Q2_K_S | 4.77GB | ✅ Available | 🟢 Yes | 📦 No
|
||||
| [Phi-3-medium-128k-instruct.IQ4_NL.gguf](https://huggingface.co/legraphista/Phi-3-medium-128k-instruct-IMat-GGUF/blob/main/Phi-3-medium-128k-instruct.IQ4_NL.gguf) | IQ4_NL | 7.90GB | ✅ Available | 🟢 Yes | 📦 No
|
||||
| [Phi-3-medium-128k-instruct.IQ4_XS.gguf](https://huggingface.co/legraphista/Phi-3-medium-128k-instruct-IMat-GGUF/blob/main/Phi-3-medium-128k-instruct.IQ4_XS.gguf) | IQ4_XS | 7.47GB | ✅ Available | 🟢 Yes | 📦 No
|
||||
| [Phi-3-medium-128k-instruct.IQ3_M.gguf](https://huggingface.co/legraphista/Phi-3-medium-128k-instruct-IMat-GGUF/blob/main/Phi-3-medium-128k-instruct.IQ3_M.gguf) | IQ3_M | 6.47GB | ✅ Available | 🟢 Yes | 📦 No
|
||||
| [Phi-3-medium-128k-instruct.IQ3_S.gguf](https://huggingface.co/legraphista/Phi-3-medium-128k-instruct-IMat-GGUF/blob/main/Phi-3-medium-128k-instruct.IQ3_S.gguf) | IQ3_S | 6.06GB | ✅ Available | 🟢 Yes | 📦 No
|
||||
| [Phi-3-medium-128k-instruct.IQ3_XS.gguf](https://huggingface.co/legraphista/Phi-3-medium-128k-instruct-IMat-GGUF/blob/main/Phi-3-medium-128k-instruct.IQ3_XS.gguf) | IQ3_XS | 5.81GB | ✅ Available | 🟢 Yes | 📦 No
|
||||
| [Phi-3-medium-128k-instruct.IQ3_XXS.gguf](https://huggingface.co/legraphista/Phi-3-medium-128k-instruct-IMat-GGUF/blob/main/Phi-3-medium-128k-instruct.IQ3_XXS.gguf) | IQ3_XXS | 5.45GB | ✅ Available | 🟢 Yes | 📦 No
|
||||
| [Phi-3-medium-128k-instruct.IQ2_M.gguf](https://huggingface.co/legraphista/Phi-3-medium-128k-instruct-IMat-GGUF/blob/main/Phi-3-medium-128k-instruct.IQ2_M.gguf) | IQ2_M | 4.72GB | ✅ Available | 🟢 Yes | 📦 No
|
||||
| [Phi-3-medium-128k-instruct.IQ2_S.gguf](https://huggingface.co/legraphista/Phi-3-medium-128k-instruct-IMat-GGUF/blob/main/Phi-3-medium-128k-instruct.IQ2_S.gguf) | IQ2_S | 4.34GB | ✅ Available | 🟢 Yes | 📦 No
|
||||
| [Phi-3-medium-128k-instruct.IQ2_XS.gguf](https://huggingface.co/legraphista/Phi-3-medium-128k-instruct-IMat-GGUF/blob/main/Phi-3-medium-128k-instruct.IQ2_XS.gguf) | IQ2_XS | 4.13GB | ✅ Available | 🟢 Yes | 📦 No
|
||||
| [Phi-3-medium-128k-instruct.IQ2_XXS.gguf](https://huggingface.co/legraphista/Phi-3-medium-128k-instruct-IMat-GGUF/blob/main/Phi-3-medium-128k-instruct.IQ2_XXS.gguf) | IQ2_XXS | 3.72GB | ✅ Available | 🟢 Yes | 📦 No
|
||||
| [Phi-3-medium-128k-instruct.IQ1_M.gguf](https://huggingface.co/legraphista/Phi-3-medium-128k-instruct-IMat-GGUF/blob/main/Phi-3-medium-128k-instruct.IQ1_M.gguf) | IQ1_M | 3.24GB | ✅ Available | 🟢 Yes | 📦 No
|
||||
| [Phi-3-medium-128k-instruct.IQ1_S.gguf](https://huggingface.co/legraphista/Phi-3-medium-128k-instruct-IMat-GGUF/blob/main/Phi-3-medium-128k-instruct.IQ1_S.gguf) | IQ1_S | 2.96GB | ✅ Available | 🟢 Yes | 📦 No
|
||||
|
||||
|
||||
## Downloading using huggingface-cli
|
||||
If you do not have hugginface-cli installed:
|
||||
```
|
||||
pip install -U "huggingface_hub[cli]"
|
||||
```
|
||||
Download the specific file you want:
|
||||
```
|
||||
huggingface-cli download legraphista/Phi-3-medium-128k-instruct-IMat-GGUF --include "Phi-3-medium-128k-instruct.Q8_0.gguf" --local-dir ./
|
||||
```
|
||||
If the model file is big, it has been split into multiple files. In order to download them all to a local folder, run:
|
||||
```
|
||||
huggingface-cli download legraphista/Phi-3-medium-128k-instruct-IMat-GGUF --include "Phi-3-medium-128k-instruct.Q8_0/*" --local-dir Phi-3-medium-128k-instruct.Q8_0
|
||||
# see FAQ for merging GGUF's
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Inference
|
||||
|
||||
### Simple chat template
|
||||
```
|
||||
<|user|>
|
||||
Can you provide ways to eat combinations of bananas and dragonfruits?<|end|>
|
||||
<|assistant|>
|
||||
Sure! Here are some ways to eat bananas and dragonfruits together:
|
||||
1. Banana and dragonfruit smoothie: Blend bananas and dragonfruits together with some milk and honey.
|
||||
2. Banana and dragonfruit salad: Mix sliced bananas and dragonfruits together with some lemon juice and honey.<|end|>
|
||||
<|user|>
|
||||
What about solving an 2x + 3 = 7 equation?<|end|>
|
||||
<|assistant|>
|
||||
|
||||
```
|
||||
|
||||
### Llama.cpp
|
||||
```
|
||||
llama.cpp/main -m Phi-3-medium-128k-instruct.Q8_0.gguf --color -i -p "prompt here (according to the chat template)"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## FAQ
|
||||
|
||||
### Why is the IMatrix not applied everywhere?
|
||||
According to [this investigation](https://www.reddit.com/r/LocalLLaMA/comments/1993iro/ggufs_quants_can_punch_above_their_weights_now/), it appears that lower quantizations are the only ones that benefit from the imatrix input (as per hellaswag results).
|
||||
|
||||
### How do I merge a split GGUF?
|
||||
1. Make sure you have `gguf-split` available
|
||||
- To get hold of `gguf-split`, navigate to https://github.com/ggerganov/llama.cpp/releases
|
||||
- Download the appropriate zip for your system from the latest release
|
||||
- Unzip the archive and you should be able to find `gguf-split`
|
||||
2. Locate your GGUF chunks folder (ex: `Phi-3-medium-128k-instruct.Q8_0`)
|
||||
3. Run `gguf-split --merge Phi-3-medium-128k-instruct.Q8_0/Phi-3-medium-128k-instruct.Q8_0-00001-of-XXXXX.gguf Phi-3-medium-128k-instruct.Q8_0.gguf`
|
||||
- Make sure to point `gguf-split` to the first chunk of the split.
|
||||
|
||||
---
|
||||
|
||||
Got a suggestion? Ping me [@legraphista](https://x.com/legraphista)!
|
||||
3
imatrix.dat
Normal file
3
imatrix.dat
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:317dad6db03712e41e39e65f4e658668ed03ff1b7dc4b5c5c40e84e7d4cecbde
|
||||
size 5330304
|
||||
3142
imatrix.dataset
Normal file
3142
imatrix.dataset
Normal file
File diff suppressed because one or more lines are too long
167
imatrix.log
Normal file
167
imatrix.log
Normal file
@@ -0,0 +1,167 @@
|
||||
main: build = 3003 (d298382a)
|
||||
main: built with cc (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0 for x86_64-linux-gnu
|
||||
main: seed = 1716755471
|
||||
llama_model_loader: loaded meta data with 27 key-value pairs and 245 tensors from Phi-3-medium-128k-instruct-IMat-GGUF/Phi-3-medium-128k-instruct.gguf (version GGUF V3 (latest))
|
||||
llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.
|
||||
llama_model_loader: - kv 0: general.architecture str = phi3
|
||||
llama_model_loader: - kv 1: general.name str = Phi3
|
||||
llama_model_loader: - kv 2: phi3.context_length u32 = 131072
|
||||
llama_model_loader: - kv 3: phi3.rope.scaling.original_context_length u32 = 4096
|
||||
llama_model_loader: - kv 4: phi3.embedding_length u32 = 5120
|
||||
llama_model_loader: - kv 5: phi3.feed_forward_length u32 = 17920
|
||||
llama_model_loader: - kv 6: phi3.block_count u32 = 40
|
||||
llama_model_loader: - kv 7: phi3.attention.head_count u32 = 40
|
||||
llama_model_loader: - kv 8: phi3.attention.head_count_kv u32 = 10
|
||||
llama_model_loader: - kv 9: phi3.attention.layer_norm_rms_epsilon f32 = 0.000010
|
||||
llama_model_loader: - kv 10: phi3.rope.dimension_count u32 = 128
|
||||
llama_model_loader: - kv 11: phi3.rope.freq_base f32 = 10000.000000
|
||||
llama_model_loader: - kv 12: general.file_type u32 = 0
|
||||
llama_model_loader: - kv 13: phi3.rope.scaling.attn_factor f32 = 1.190238
|
||||
llama_model_loader: - kv 14: tokenizer.ggml.model str = llama
|
||||
llama_model_loader: - kv 15: tokenizer.ggml.pre str = default
|
||||
llama_model_loader: - kv 16: tokenizer.ggml.tokens arr[str,32064] = ["<unk>", "<s>", "</s>", "<0x00>", "<...
|
||||
llama_model_loader: - kv 17: tokenizer.ggml.scores arr[f32,32064] = [-1000.000000, -1000.000000, -1000.00...
|
||||
llama_model_loader: - kv 18: tokenizer.ggml.token_type arr[i32,32064] = [3, 3, 4, 6, 6, 6, 6, 6, 6, 6, 6, 6, ...
|
||||
llama_model_loader: - kv 19: tokenizer.ggml.bos_token_id u32 = 1
|
||||
llama_model_loader: - kv 20: tokenizer.ggml.eos_token_id u32 = 32000
|
||||
llama_model_loader: - kv 21: tokenizer.ggml.unknown_token_id u32 = 0
|
||||
llama_model_loader: - kv 22: tokenizer.ggml.padding_token_id u32 = 32000
|
||||
llama_model_loader: - kv 23: tokenizer.ggml.add_bos_token bool = false
|
||||
llama_model_loader: - kv 24: tokenizer.ggml.add_eos_token bool = false
|
||||
llama_model_loader: - kv 25: tokenizer.chat_template str = {% for message in messages %}{% if (m...
|
||||
llama_model_loader: - kv 26: general.quantization_version u32 = 2
|
||||
llama_model_loader: - type f32: 245 tensors
|
||||
llm_load_vocab: special tokens definition check successful ( 323/32064 ).
|
||||
llm_load_print_meta: format = GGUF V3 (latest)
|
||||
llm_load_print_meta: arch = phi3
|
||||
llm_load_print_meta: vocab type = SPM
|
||||
llm_load_print_meta: n_vocab = 32064
|
||||
llm_load_print_meta: n_merges = 0
|
||||
llm_load_print_meta: n_ctx_train = 131072
|
||||
llm_load_print_meta: n_embd = 5120
|
||||
llm_load_print_meta: n_head = 40
|
||||
llm_load_print_meta: n_head_kv = 10
|
||||
llm_load_print_meta: n_layer = 40
|
||||
llm_load_print_meta: n_rot = 128
|
||||
llm_load_print_meta: n_embd_head_k = 128
|
||||
llm_load_print_meta: n_embd_head_v = 128
|
||||
llm_load_print_meta: n_gqa = 4
|
||||
llm_load_print_meta: n_embd_k_gqa = 1280
|
||||
llm_load_print_meta: n_embd_v_gqa = 1280
|
||||
llm_load_print_meta: f_norm_eps = 0.0e+00
|
||||
llm_load_print_meta: f_norm_rms_eps = 1.0e-05
|
||||
llm_load_print_meta: f_clamp_kqv = 0.0e+00
|
||||
llm_load_print_meta: f_max_alibi_bias = 0.0e+00
|
||||
llm_load_print_meta: f_logit_scale = 0.0e+00
|
||||
llm_load_print_meta: n_ff = 17920
|
||||
llm_load_print_meta: n_expert = 0
|
||||
llm_load_print_meta: n_expert_used = 0
|
||||
llm_load_print_meta: causal attn = 1
|
||||
llm_load_print_meta: pooling type = 0
|
||||
llm_load_print_meta: rope type = 2
|
||||
llm_load_print_meta: rope scaling = linear
|
||||
llm_load_print_meta: freq_base_train = 10000.0
|
||||
llm_load_print_meta: freq_scale_train = 1
|
||||
llm_load_print_meta: n_yarn_orig_ctx = 4096
|
||||
llm_load_print_meta: rope_finetuned = unknown
|
||||
llm_load_print_meta: ssm_d_conv = 0
|
||||
llm_load_print_meta: ssm_d_inner = 0
|
||||
llm_load_print_meta: ssm_d_state = 0
|
||||
llm_load_print_meta: ssm_dt_rank = 0
|
||||
llm_load_print_meta: model type = 14B
|
||||
llm_load_print_meta: model ftype = all F32
|
||||
llm_load_print_meta: model params = 13.96 B
|
||||
llm_load_print_meta: model size = 52.01 GiB (32.00 BPW)
|
||||
llm_load_print_meta: general.name = Phi3
|
||||
llm_load_print_meta: BOS token = 1 '<s>'
|
||||
llm_load_print_meta: EOS token = 32000 '<|endoftext|>'
|
||||
llm_load_print_meta: UNK token = 0 '<unk>'
|
||||
llm_load_print_meta: PAD token = 32000 '<|endoftext|>'
|
||||
llm_load_print_meta: LF token = 13 '<0x0A>'
|
||||
llm_load_print_meta: EOT token = 32007 '<|end|>'
|
||||
ggml_cuda_init: GGML_CUDA_FORCE_MMQ: no
|
||||
ggml_cuda_init: CUDA_USE_TENSOR_CORES: yes
|
||||
ggml_cuda_init: found 1 CUDA devices:
|
||||
Device 0: NVIDIA GeForce RTX 4090, compute capability 8.9, VMM: yes
|
||||
llm_load_tensors: ggml ctx size = 0.28 MiB
|
||||
llm_load_tensors: offloading 16 repeating layers to GPU
|
||||
llm_load_tensors: offloaded 16/41 layers to GPU
|
||||
llm_load_tensors: CPU buffer size = 53254.08 MiB
|
||||
llm_load_tensors: CUDA0 buffer size = 20800.63 MiB
|
||||
.......................................................................................
|
||||
llama_new_context_with_model: n_ctx = 512
|
||||
llama_new_context_with_model: n_batch = 512
|
||||
llama_new_context_with_model: n_ubatch = 512
|
||||
llama_new_context_with_model: flash_attn = 0
|
||||
llama_new_context_with_model: freq_base = 10000.0
|
||||
llama_new_context_with_model: freq_scale = 1
|
||||
llama_kv_cache_init: CUDA_Host KV buffer size = 60.00 MiB
|
||||
llama_kv_cache_init: CUDA0 KV buffer size = 40.00 MiB
|
||||
llama_new_context_with_model: KV self size = 100.00 MiB, K (f16): 50.00 MiB, V (f16): 50.00 MiB
|
||||
llama_new_context_with_model: CUDA_Host output buffer size = 0.12 MiB
|
||||
llama_new_context_with_model: CUDA0 compute buffer size = 840.00 MiB
|
||||
llama_new_context_with_model: CUDA_Host compute buffer size = 20.98 MiB
|
||||
llama_new_context_with_model: graph nodes = 1606
|
||||
llama_new_context_with_model: graph splits = 220
|
||||
|
||||
system_info: n_threads = 25 / 32 | AVX = 1 | AVX_VNNI = 0 | AVX2 = 1 | AVX512 = 1 | AVX512_VBMI = 1 | AVX512_VNNI = 1 | AVX512_BF16 = 1 | FMA = 1 | NEON = 0 | SVE = 0 | ARM_FMA = 0 | F16C = 1 | FP16_VA = 0 | WASM_SIMD = 0 | BLAS = 1 | SSE3 = 1 | SSSE3 = 1 | VSX = 0 | MATMUL_INT8 = 0 | LLAMAFILE = 1 |
|
||||
compute_imatrix: tokenizing the input ..
|
||||
compute_imatrix: tokenization took 133.033 ms
|
||||
compute_imatrix: computing over 234 chunks with batch_size 512
|
||||
compute_imatrix: 2.40 seconds per pass - ETA 9.37 minutes
|
||||
[1]4.4810,[2]3.4854,[3]3.4301,[4]3.6863,[5]4.1180,[6]4.2207,[7]3.8129,[8]4.1860,[9]4.4113,
|
||||
save_imatrix: stored collected data after 10 chunks in Phi-3-medium-128k-instruct-IMat-GGUF/imatrix.dat
|
||||
[10]4.6985,[11]4.7333,[12]4.4193,[13]4.5511,[14]4.4712,[15]4.8532,[16]4.9821,[17]5.2736,[18]5.4072,[19]5.5905,
|
||||
save_imatrix: stored collected data after 20 chunks in Phi-3-medium-128k-instruct-IMat-GGUF/imatrix.dat
|
||||
[20]5.7241,[21]5.8095,[22]5.9993,[23]5.7705,[24]5.6384,[25]5.6408,[26]5.3666,[27]5.1654,[28]4.9147,[29]4.8876,
|
||||
save_imatrix: stored collected data after 30 chunks in Phi-3-medium-128k-instruct-IMat-GGUF/imatrix.dat
|
||||
[30]4.9822,[31]5.0317,[32]5.0841,[33]5.0646,[34]5.0884,[35]5.0893,[36]4.9139,[37]4.7968,[38]4.7524,[39]4.7424,
|
||||
save_imatrix: stored collected data after 40 chunks in Phi-3-medium-128k-instruct-IMat-GGUF/imatrix.dat
|
||||
[40]4.7279,[41]4.6708,[42]4.7080,[43]4.7361,[44]4.7795,[45]4.8462,[46]4.9090,[47]4.9759,[48]5.0986,[49]5.1941,
|
||||
save_imatrix: stored collected data after 50 chunks in Phi-3-medium-128k-instruct-IMat-GGUF/imatrix.dat
|
||||
[50]5.2907,[51]5.3709,[52]5.4552,[53]5.4307,[54]5.3597,[55]5.3080,[56]5.3784,[57]5.4179,[58]5.4352,[59]5.4922,
|
||||
save_imatrix: stored collected data after 60 chunks in Phi-3-medium-128k-instruct-IMat-GGUF/imatrix.dat
|
||||
[60]5.5560,[61]5.5745,[62]5.6139,[63]5.6415,[64]5.6889,[65]5.7133,[66]5.7433,[67]5.7732,[68]5.8083,[69]5.8716,
|
||||
save_imatrix: stored collected data after 70 chunks in Phi-3-medium-128k-instruct-IMat-GGUF/imatrix.dat
|
||||
[70]5.9098,[71]5.9411,[72]5.9714,[73]5.9286,[74]5.8875,[75]5.7936,[76]5.7094,[77]5.6668,[78]5.5887,[79]5.5385,
|
||||
save_imatrix: stored collected data after 80 chunks in Phi-3-medium-128k-instruct-IMat-GGUF/imatrix.dat
|
||||
[80]5.4780,[81]5.4016,[82]5.3362,[83]5.2999,[84]5.2918,[85]5.3183,[86]5.3337,[87]5.3620,[88]5.3804,[89]5.3639,
|
||||
save_imatrix: stored collected data after 90 chunks in Phi-3-medium-128k-instruct-IMat-GGUF/imatrix.dat
|
||||
[90]5.2972,[91]5.3001,[92]5.2878,[93]5.2948,[94]5.2938,[95]5.3006,[96]5.3134,[97]5.3192,[98]5.3045,[99]5.2758,
|
||||
save_imatrix: stored collected data after 100 chunks in Phi-3-medium-128k-instruct-IMat-GGUF/imatrix.dat
|
||||
[100]5.2885,[101]5.3082,[102]5.3067,[103]5.2856,[104]5.2458,[105]5.2354,[106]5.2487,[107]5.2657,[108]5.2514,[109]5.2471,
|
||||
save_imatrix: stored collected data after 110 chunks in Phi-3-medium-128k-instruct-IMat-GGUF/imatrix.dat
|
||||
[110]5.2362,[111]5.2459,[112]5.2581,[113]5.2588,[114]5.2725,[115]5.2767,[116]5.2727,[117]5.2718,[118]5.2804,[119]5.2674,
|
||||
save_imatrix: stored collected data after 120 chunks in Phi-3-medium-128k-instruct-IMat-GGUF/imatrix.dat
|
||||
[120]5.2713,[121]5.2633,[122]5.2464,[123]5.2637,[124]5.2612,[125]5.2696,[126]5.2574,[127]5.2593,[128]5.2702,[129]5.2540,
|
||||
save_imatrix: stored collected data after 130 chunks in Phi-3-medium-128k-instruct-IMat-GGUF/imatrix.dat
|
||||
[130]5.2362,[131]5.2287,[132]5.2281,[133]5.1918,[134]5.1895,[135]5.1684,[136]5.1480,[137]5.1234,[138]5.0985,[139]5.0746,
|
||||
save_imatrix: stored collected data after 140 chunks in Phi-3-medium-128k-instruct-IMat-GGUF/imatrix.dat
|
||||
[140]5.0538,[141]5.0340,[142]5.0132,[143]5.0052,[144]4.9975,[145]4.9791,[146]4.9583,[147]4.9526,[148]4.9368,[149]4.9266,
|
||||
save_imatrix: stored collected data after 150 chunks in Phi-3-medium-128k-instruct-IMat-GGUF/imatrix.dat
|
||||
[150]4.9150,[151]4.9036,[152]4.8937,[153]4.8784,[154]4.8674,[155]4.8813,[156]4.8594,[157]4.8542,[158]4.8593,[159]4.8556,
|
||||
save_imatrix: stored collected data after 160 chunks in Phi-3-medium-128k-instruct-IMat-GGUF/imatrix.dat
|
||||
[160]4.8535,[161]4.8535,[162]4.8522,[163]4.8599,[164]4.8650,[165]4.8772,[166]4.8805,[167]4.8787,[168]4.8838,[169]4.8911,
|
||||
save_imatrix: stored collected data after 170 chunks in Phi-3-medium-128k-instruct-IMat-GGUF/imatrix.dat
|
||||
[170]4.9040,[171]4.9000,[172]4.9053,[173]4.9210,[174]4.9268,[175]4.9438,[176]4.9541,[177]4.9646,[178]4.9726,[179]4.9948,
|
||||
save_imatrix: stored collected data after 180 chunks in Phi-3-medium-128k-instruct-IMat-GGUF/imatrix.dat
|
||||
[180]5.0044,[181]5.0426,[182]5.0583,[183]5.0790,[184]5.0869,[185]5.0948,[186]5.1043,[187]5.1094,[188]5.1017,[189]5.1070,
|
||||
save_imatrix: stored collected data after 190 chunks in Phi-3-medium-128k-instruct-IMat-GGUF/imatrix.dat
|
||||
[190]5.1140,[191]5.1248,[192]5.1314,[193]5.1552,[194]5.1474,[195]5.1256,[196]5.1589,[197]5.1894,[198]5.2157,[199]5.2588,
|
||||
save_imatrix: stored collected data after 200 chunks in Phi-3-medium-128k-instruct-IMat-GGUF/imatrix.dat
|
||||
[200]5.2955,[201]5.3054,[202]5.3112,[203]5.2801,[204]5.2823,[205]5.2896,[206]5.3126,[207]5.3107,[208]5.3150,[209]5.3179,
|
||||
save_imatrix: stored collected data after 210 chunks in Phi-3-medium-128k-instruct-IMat-GGUF/imatrix.dat
|
||||
[210]5.3271,[211]5.3399,[212]5.3411,[213]5.3404,[214]5.3484,[215]5.3635,[216]5.3794,[217]5.3830,[218]5.3796,[219]5.3782,
|
||||
save_imatrix: stored collected data after 220 chunks in Phi-3-medium-128k-instruct-IMat-GGUF/imatrix.dat
|
||||
[220]5.3737,[221]5.3749,[222]5.3753,[223]5.3841,[224]5.3695,[225]5.3727,[226]5.3613,[227]5.3894,[228]5.4175,[229]5.4522,
|
||||
save_imatrix: stored collected data after 230 chunks in Phi-3-medium-128k-instruct-IMat-GGUF/imatrix.dat
|
||||
[230]5.4826,[231]5.4968,[232]5.4777,[233]5.4565,[234]5.4323,
|
||||
save_imatrix: stored collected data after 234 chunks in Phi-3-medium-128k-instruct-IMat-GGUF/imatrix.dat
|
||||
|
||||
llama_print_timings: load time = 5039.95 ms
|
||||
llama_print_timings: sample time = 0.00 ms / 1 runs ( 0.00 ms per token, inf tokens per second)
|
||||
llama_print_timings: prompt eval time = 543719.95 ms / 119808 tokens ( 4.54 ms per token, 220.35 tokens per second)
|
||||
llama_print_timings: eval time = 0.00 ms / 1 runs ( 0.00 ms per token, inf tokens per second)
|
||||
llama_print_timings: total time = 547185.12 ms / 119809 tokens
|
||||
|
||||
Final estimate: PPL = 5.4323 +/- 0.05178
|
||||
Reference in New Issue
Block a user