bartowski/cybertron-v4-qw7B-UNAMGS-GGUF
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

初始化项目，由ModelHub XC社区提供模型

Browse Source 
Model: bartowski/cybertron-v4-qw7B-UNAMGS-GGUF
Source: Original Platform
This commit is contained in:
ModelHub XC
2026-08-05 06:15:13 +08:00
commit 5ad6949a19
28 changed files with 420 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

60
.gitattributes vendored Normal file
View File

@@ -0,0 +1,60 @@
*.7z filter=lfs diff=lfs merge=lfs -text
*.arrow filter=lfs diff=lfs merge=lfs -text
*.bin filter=lfs diff=lfs merge=lfs -text
*.bz2 filter=lfs diff=lfs merge=lfs -text
*.ckpt filter=lfs diff=lfs merge=lfs -text
*.ftz filter=lfs diff=lfs merge=lfs -text
*.gz filter=lfs diff=lfs merge=lfs -text
*.h5 filter=lfs diff=lfs merge=lfs -text
*.joblib filter=lfs diff=lfs merge=lfs -text
*.lfs.* filter=lfs diff=lfs merge=lfs -text
*.mlmodel filter=lfs diff=lfs merge=lfs -text
*.model filter=lfs diff=lfs merge=lfs -text
*.msgpack filter=lfs diff=lfs merge=lfs -text
*.npy filter=lfs diff=lfs merge=lfs -text
*.npz filter=lfs diff=lfs merge=lfs -text
*.onnx filter=lfs diff=lfs merge=lfs -text
*.ot filter=lfs diff=lfs merge=lfs -text
*.parquet filter=lfs diff=lfs merge=lfs -text
*.pb filter=lfs diff=lfs merge=lfs -text
*.pickle filter=lfs diff=lfs merge=lfs -text
*.pkl filter=lfs diff=lfs merge=lfs -text
*.pt filter=lfs diff=lfs merge=lfs -text
*.pth filter=lfs diff=lfs merge=lfs -text
*.rar filter=lfs diff=lfs merge=lfs -text
*.safetensors filter=lfs diff=lfs merge=lfs -text
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
*.tar.* filter=lfs diff=lfs merge=lfs -text
*.tar filter=lfs diff=lfs merge=lfs -text
*.tflite filter=lfs diff=lfs merge=lfs -text
*.tgz filter=lfs diff=lfs merge=lfs -text
*.wasm filter=lfs diff=lfs merge=lfs -text
*.xz filter=lfs diff=lfs merge=lfs -text
*.zip filter=lfs diff=lfs merge=lfs -text
*.zst filter=lfs diff=lfs merge=lfs -text
*tfevents* filter=lfs diff=lfs merge=lfs -text
cybertron-v4-qw7B-UNAMGS-Q8_0.gguf filter=lfs diff=lfs merge=lfs -text
cybertron-v4-qw7B-UNAMGS-Q6_K_L.gguf filter=lfs diff=lfs merge=lfs -text
cybertron-v4-qw7B-UNAMGS-Q6_K.gguf filter=lfs diff=lfs merge=lfs -text
cybertron-v4-qw7B-UNAMGS-Q5_K_L.gguf filter=lfs diff=lfs merge=lfs -text
cybertron-v4-qw7B-UNAMGS-Q5_K_M.gguf filter=lfs diff=lfs merge=lfs -text
cybertron-v4-qw7B-UNAMGS-Q5_K_S.gguf filter=lfs diff=lfs merge=lfs -text
cybertron-v4-qw7B-UNAMGS-Q4_K_L.gguf filter=lfs diff=lfs merge=lfs -text
cybertron-v4-qw7B-UNAMGS-Q4_K_M.gguf filter=lfs diff=lfs merge=lfs -text
cybertron-v4-qw7B-UNAMGS-Q4_K_S.gguf filter=lfs diff=lfs merge=lfs -text
cybertron-v4-qw7B-UNAMGS-Q4_0_8_8.gguf filter=lfs diff=lfs merge=lfs -text
cybertron-v4-qw7B-UNAMGS-Q4_0_4_8.gguf filter=lfs diff=lfs merge=lfs -text
cybertron-v4-qw7B-UNAMGS-Q4_0_4_4.gguf filter=lfs diff=lfs merge=lfs -text
cybertron-v4-qw7B-UNAMGS-Q4_0.gguf filter=lfs diff=lfs merge=lfs -text
cybertron-v4-qw7B-UNAMGS-IQ4_XS.gguf filter=lfs diff=lfs merge=lfs -text
cybertron-v4-qw7B-UNAMGS-Q3_K_XL.gguf filter=lfs diff=lfs merge=lfs -text
cybertron-v4-qw7B-UNAMGS-Q3_K_L.gguf filter=lfs diff=lfs merge=lfs -text
cybertron-v4-qw7B-UNAMGS-Q3_K_M.gguf filter=lfs diff=lfs merge=lfs -text
cybertron-v4-qw7B-UNAMGS-IQ3_M.gguf filter=lfs diff=lfs merge=lfs -text
cybertron-v4-qw7B-UNAMGS-Q3_K_S.gguf filter=lfs diff=lfs merge=lfs -text
cybertron-v4-qw7B-UNAMGS-IQ3_XS.gguf filter=lfs diff=lfs merge=lfs -text
cybertron-v4-qw7B-UNAMGS-Q2_K_L.gguf filter=lfs diff=lfs merge=lfs -text
cybertron-v4-qw7B-UNAMGS-Q2_K.gguf filter=lfs diff=lfs merge=lfs -text
cybertron-v4-qw7B-UNAMGS-IQ2_M.gguf filter=lfs diff=lfs merge=lfs -text
cybertron-v4-qw7B-UNAMGS-f16.gguf filter=lfs diff=lfs merge=lfs -text
cybertron-v4-qw7B-UNAMGS.imatrix filter=lfs diff=lfs merge=lfs -text

284
README.md Normal file
View File

@@ -0,0 +1,284 @@
---
quantized_by: bartowski
pipeline_tag: text-generation
datasets:
- Magpie-Align/Magpie-Qwen2.5-Pro-1M-v0.1
base_model: fblgit/cybertron-v4-qw7B-UNAMGS
license_name: qwen
tags:
- generated_from_trainer
license: other
language:
- en
license_link: https://huggingface.co/Qwen/Qwen2.5-72B-Instruct/blob/main/LICENSE
model-index:
- name: cybertron-v4-qw7B-UNAMGS
results:
- task:
type: text-generation
name: Text Generation
dataset:
name: IFEval (0-Shot)
type: HuggingFaceH4/ifeval
args:
num_few_shot: 0
metrics:
- type: inst_level_strict_acc and prompt_level_strict_acc
value: 60.84
name: strict accuracy
source:
url: https://huggingface.co/spaces/open-llm-leaderboard/open_llm_leaderboard?query=fblgit/cybertron-v4-qw7B-UNAMGS
name: Open LLM Leaderboard
- task:
type: text-generation
name: Text Generation
dataset:
name: BBH (3-Shot)
type: BBH
args:
num_few_shot: 3
metrics:
- type: acc_norm
value: 37.71
name: normalized accuracy
source:
url: https://huggingface.co/spaces/open-llm-leaderboard/open_llm_leaderboard?query=fblgit/cybertron-v4-qw7B-UNAMGS
name: Open LLM Leaderboard
- task:
type: text-generation
name: Text Generation
dataset:
name: MATH Lvl 5 (4-Shot)
type: hendrycks/competition_math
args:
num_few_shot: 4
metrics:
- type: exact_match
value: 29.91
name: exact match
source:
url: https://huggingface.co/spaces/open-llm-leaderboard/open_llm_leaderboard?query=fblgit/cybertron-v4-qw7B-UNAMGS
name: Open LLM Leaderboard
- task:
type: text-generation
name: Text Generation
dataset:
name: GPQA (0-shot)
type: Idavidrein/gpqa
args:
num_few_shot: 0
metrics:
- type: acc_norm
value: 10.85
name: acc_norm
source:
url: https://huggingface.co/spaces/open-llm-leaderboard/open_llm_leaderboard?query=fblgit/cybertron-v4-qw7B-UNAMGS
name: Open LLM Leaderboard
- task:
type: text-generation
name: Text Generation
dataset:
name: MuSR (0-shot)
type: TAUR-Lab/MuSR
args:
num_few_shot: 0
metrics:
- type: acc_norm
value: 12.69
name: acc_norm
source:
url: https://huggingface.co/spaces/open-llm-leaderboard/open_llm_leaderboard?query=fblgit/cybertron-v4-qw7B-UNAMGS
name: Open LLM Leaderboard
- task:
type: text-generation
name: Text Generation
dataset:
name: MMLU-PRO (5-shot)
type: TIGER-Lab/MMLU-Pro
config: main
split: test
args:
num_few_shot: 5
metrics:
- type: acc
value: 38.89
name: accuracy
source:
url: https://huggingface.co/spaces/open-llm-leaderboard/open_llm_leaderboard?query=fblgit/cybertron-v4-qw7B-UNAMGS
name: Open LLM Leaderboard
---
## Llamacpp imatrix Quantizations of cybertron-v4-qw7B-UNAMGS
Using <a href="https://github.com/ggerganov/llama.cpp/">llama.cpp</a> release <a href="https://github.com/ggerganov/llama.cpp/releases/tag/b4132">b4132</a> for quantization.
Original model: https://huggingface.co/fblgit/cybertron-v4-qw7B-UNAMGS
# cybertron-v4-qw7B-UNAMGS
**UNA IS BACK** Cybertron v4 UNA-MGS, Based on the amazing Qwen2.5 7B
**SCORING #1 7-8B LLM WITH NO CONTAMINATION 21.11.2024 with avg. 31.82**
![cybertron-v4-MGS](https://huggingface.co/fblgit/cybertron-v4-qw7B-MGS/resolve/main/cybertron_v4MGS.png)
This special edition went thru UNA at MLP layers just like [miniclaus-1.5B](https://huggingface.co/fblgit/miniclaus-qw1.5B-UNAMGS)
Here we use our novel approach called `MGS`. Its up to you to figure out what it means. On top of that we used `UNA: Uniform Neural Alignment`
Cybertron V4 went thru SFT with `MGS & UNA` over `Magpie-Align/Magpie-Qwen2.5-Pro-1M-v0.1` dataset.
All quants made using imatrix option with dataset from [here](https://gist.github.com/bartowski1182/eb213dccb3571f863da82e99418f81e8)
Run them in [LM Studio](https://lmstudio.ai/)
## Prompt format
```
<|im_start|>system
{system_prompt}<|im_end|>
<|im_start|>user
{prompt}<|im_end|>
<|im_start|>assistant
```
## Download a file (not the whole branch) from below:
| Filename | Quant type | File Size | Split | Description |
| -------- | ---------- | --------- | ----- | ----------- |
| [cybertron-v4-qw7B-UNAMGS-f16.gguf](https://huggingface.co/bartowski/cybertron-v4-qw7B-UNAMGS-GGUF/blob/main/cybertron-v4-qw7B-UNAMGS-f16.gguf) | f16 | 15.24GB | false | Full F16 weights. |
| [cybertron-v4-qw7B-UNAMGS-Q8_0.gguf](https://huggingface.co/bartowski/cybertron-v4-qw7B-UNAMGS-GGUF/blob/main/cybertron-v4-qw7B-UNAMGS-Q8_0.gguf) | Q8_0 | 8.10GB | false | Extremely high quality, generally unneeded but max available quant. |
| [cybertron-v4-qw7B-UNAMGS-Q6_K_L.gguf](https://huggingface.co/bartowski/cybertron-v4-qw7B-UNAMGS-GGUF/blob/main/cybertron-v4-qw7B-UNAMGS-Q6_K_L.gguf) | Q6_K_L | 6.52GB | false | Uses Q8_0 for embed and output weights. Very high quality, near perfect, *recommended*. |
| [cybertron-v4-qw7B-UNAMGS-Q6_K.gguf](https://huggingface.co/bartowski/cybertron-v4-qw7B-UNAMGS-GGUF/blob/main/cybertron-v4-qw7B-UNAMGS-Q6_K.gguf) | Q6_K | 6.25GB | false | Very high quality, near perfect, *recommended*. |
| [cybertron-v4-qw7B-UNAMGS-Q5_K_L.gguf](https://huggingface.co/bartowski/cybertron-v4-qw7B-UNAMGS-GGUF/blob/main/cybertron-v4-qw7B-UNAMGS-Q5_K_L.gguf) | Q5_K_L | 5.78GB | false | Uses Q8_0 for embed and output weights. High quality, *recommended*. |
| [cybertron-v4-qw7B-UNAMGS-Q5_K_M.gguf](https://huggingface.co/bartowski/cybertron-v4-qw7B-UNAMGS-GGUF/blob/main/cybertron-v4-qw7B-UNAMGS-Q5_K_M.gguf) | Q5_K_M | 5.44GB | false | High quality, *recommended*. |
| [cybertron-v4-qw7B-UNAMGS-Q5_K_S.gguf](https://huggingface.co/bartowski/cybertron-v4-qw7B-UNAMGS-GGUF/blob/main/cybertron-v4-qw7B-UNAMGS-Q5_K_S.gguf) | Q5_K_S | 5.32GB | false | High quality, *recommended*. |
| [cybertron-v4-qw7B-UNAMGS-Q4_K_L.gguf](https://huggingface.co/bartowski/cybertron-v4-qw7B-UNAMGS-GGUF/blob/main/cybertron-v4-qw7B-UNAMGS-Q4_K_L.gguf) | Q4_K_L | 5.09GB | false | Uses Q8_0 for embed and output weights. Good quality, *recommended*. |
| [cybertron-v4-qw7B-UNAMGS-Q4_K_M.gguf](https://huggingface.co/bartowski/cybertron-v4-qw7B-UNAMGS-GGUF/blob/main/cybertron-v4-qw7B-UNAMGS-Q4_K_M.gguf) | Q4_K_M | 4.68GB | false | Good quality, default size for most use cases, *recommended*. |
| [cybertron-v4-qw7B-UNAMGS-Q3_K_XL.gguf](https://huggingface.co/bartowski/cybertron-v4-qw7B-UNAMGS-GGUF/blob/main/cybertron-v4-qw7B-UNAMGS-Q3_K_XL.gguf) | Q3_K_XL | 4.57GB | false | Uses Q8_0 for embed and output weights. Lower quality but usable, good for low RAM availability. |
| [cybertron-v4-qw7B-UNAMGS-Q4_K_S.gguf](https://huggingface.co/bartowski/cybertron-v4-qw7B-UNAMGS-GGUF/blob/main/cybertron-v4-qw7B-UNAMGS-Q4_K_S.gguf) | Q4_K_S | 4.46GB | false | Slightly lower quality with more space savings, *recommended*. |
| [cybertron-v4-qw7B-UNAMGS-Q4_0.gguf](https://huggingface.co/bartowski/cybertron-v4-qw7B-UNAMGS-GGUF/blob/main/cybertron-v4-qw7B-UNAMGS-Q4_0.gguf) | Q4_0 | 4.44GB | false | Legacy format, generally not worth using over similarly sized formats |
| [cybertron-v4-qw7B-UNAMGS-Q4_0_8_8.gguf](https://huggingface.co/bartowski/cybertron-v4-qw7B-UNAMGS-GGUF/blob/main/cybertron-v4-qw7B-UNAMGS-Q4_0_8_8.gguf) | Q4_0_8_8 | 4.43GB | false | Optimized for ARM and AVX inference. Requires 'sve' support for ARM (see details below). *Don't use on Mac*. |
| [cybertron-v4-qw7B-UNAMGS-Q4_0_4_8.gguf](https://huggingface.co/bartowski/cybertron-v4-qw7B-UNAMGS-GGUF/blob/main/cybertron-v4-qw7B-UNAMGS-Q4_0_4_8.gguf) | Q4_0_4_8 | 4.43GB | false | Optimized for ARM inference. Requires 'i8mm' support (see details below). *Don't use on Mac*. |
| [cybertron-v4-qw7B-UNAMGS-Q4_0_4_4.gguf](https://huggingface.co/bartowski/cybertron-v4-qw7B-UNAMGS-GGUF/blob/main/cybertron-v4-qw7B-UNAMGS-Q4_0_4_4.gguf) | Q4_0_4_4 | 4.43GB | false | Optimized for ARM inference. Should work well on all ARM chips, not for use with GPUs. *Don't use on Mac*. |
| [cybertron-v4-qw7B-UNAMGS-IQ4_XS.gguf](https://huggingface.co/bartowski/cybertron-v4-qw7B-UNAMGS-GGUF/blob/main/cybertron-v4-qw7B-UNAMGS-IQ4_XS.gguf) | IQ4_XS | 4.22GB | false | Decent quality, smaller than Q4_K_S with similar performance, *recommended*. |
| [cybertron-v4-qw7B-UNAMGS-Q3_K_L.gguf](https://huggingface.co/bartowski/cybertron-v4-qw7B-UNAMGS-GGUF/blob/main/cybertron-v4-qw7B-UNAMGS-Q3_K_L.gguf) | Q3_K_L | 4.09GB | false | Lower quality but usable, good for low RAM availability. |
| [cybertron-v4-qw7B-UNAMGS-Q3_K_M.gguf](https://huggingface.co/bartowski/cybertron-v4-qw7B-UNAMGS-GGUF/blob/main/cybertron-v4-qw7B-UNAMGS-Q3_K_M.gguf) | Q3_K_M | 3.81GB | false | Low quality. |
| [cybertron-v4-qw7B-UNAMGS-IQ3_M.gguf](https://huggingface.co/bartowski/cybertron-v4-qw7B-UNAMGS-GGUF/blob/main/cybertron-v4-qw7B-UNAMGS-IQ3_M.gguf) | IQ3_M | 3.57GB | false | Medium-low quality, new method with decent performance comparable to Q3_K_M. |
| [cybertron-v4-qw7B-UNAMGS-Q2_K_L.gguf](https://huggingface.co/bartowski/cybertron-v4-qw7B-UNAMGS-GGUF/blob/main/cybertron-v4-qw7B-UNAMGS-Q2_K_L.gguf) | Q2_K_L | 3.55GB | false | Uses Q8_0 for embed and output weights. Very low quality but surprisingly usable. |
| [cybertron-v4-qw7B-UNAMGS-Q3_K_S.gguf](https://huggingface.co/bartowski/cybertron-v4-qw7B-UNAMGS-GGUF/blob/main/cybertron-v4-qw7B-UNAMGS-Q3_K_S.gguf) | Q3_K_S | 3.49GB | false | Low quality, not recommended. |
| [cybertron-v4-qw7B-UNAMGS-IQ3_XS.gguf](https://huggingface.co/bartowski/cybertron-v4-qw7B-UNAMGS-GGUF/blob/main/cybertron-v4-qw7B-UNAMGS-IQ3_XS.gguf) | IQ3_XS | 3.35GB | false | Lower quality, new method with decent performance, slightly better than Q3_K_S. |
| [cybertron-v4-qw7B-UNAMGS-Q2_K.gguf](https://huggingface.co/bartowski/cybertron-v4-qw7B-UNAMGS-GGUF/blob/main/cybertron-v4-qw7B-UNAMGS-Q2_K.gguf) | Q2_K | 3.02GB | false | Very low quality but surprisingly usable. |
| [cybertron-v4-qw7B-UNAMGS-IQ2_M.gguf](https://huggingface.co/bartowski/cybertron-v4-qw7B-UNAMGS-GGUF/blob/main/cybertron-v4-qw7B-UNAMGS-IQ2_M.gguf) | IQ2_M | 2.78GB | false | Relatively low quality, uses SOTA techniques to be surprisingly usable. |
## Embed/output weights
Some of these quants (Q3_K_XL, Q4_K_L etc) are the standard quantization method with the embeddings and output weights quantized to Q8_0 instead of what they would normally default to.
## Downloading using huggingface-cli
<details>
<summary>Click to view download instructions</summary>
First, make sure you have hugginface-cli installed:
```
pip install -U "huggingface_hub[cli]"
```
Then, you can target the specific file you want:
```
huggingface-cli download bartowski/cybertron-v4-qw7B-UNAMGS-GGUF --include "cybertron-v4-qw7B-UNAMGS-Q4_K_M.gguf" --local-dir ./
```
If the model is bigger than 50GB, it will have been split into multiple files. In order to download them all to a local folder, run:
```
huggingface-cli download bartowski/cybertron-v4-qw7B-UNAMGS-GGUF --include "cybertron-v4-qw7B-UNAMGS-Q8_0/*" --local-dir ./
```
You can either specify a new local-dir (cybertron-v4-qw7B-UNAMGS-Q8_0) or download them all in place (./)
</details>
## Q4_0_X_X information
<details>
<summary>Click to view Q4_0_X_X information</summary>
These are *NOT* for Metal (Apple) or GPU (nvidia/AMD/intel) offloading, only ARM chips (and certain AVX2/AVX512 CPUs).
If you're using an ARM chip, the Q4_0_X_X quants will have a substantial speedup. Check out Q4_0_4_4 speed comparisons [on the original pull request](https://github.com/ggerganov/llama.cpp/pull/5780#pullrequestreview-21657544660)
To check which one would work best for your ARM chip, you can check [AArch64 SoC features](https://gpages.juszkiewicz.com.pl/arm-socs-table/arm-socs.html) (thanks EloyOn!).
If you're using a CPU that supports AVX2 or AVX512 (typically server CPUs and AMD's latest Zen5 CPUs) and are not offloading to a GPU, the Q4_0_8_8 may offer a nice speed as well:
<details>
<summary>Click to view benchmarks on an AVX2 system (EPYC7702)</summary>
| model | size | params | backend | threads | test | t/s | % (vs Q4_0) |
| ------------------------------ | ---------: | ---------: | ---------- | ------: | ------------: | -------------------: |-------------: |
| qwen2 3B Q4_0 | 1.70 GiB | 3.09 B | CPU | 64 | pp512 | 204.03 ± 1.03 | 100% |
| qwen2 3B Q4_0 | 1.70 GiB | 3.09 B | CPU | 64 | pp1024 | 282.92 ± 0.19 | 100% |
| qwen2 3B Q4_0 | 1.70 GiB | 3.09 B | CPU | 64 | pp2048 | 259.49 ± 0.44 | 100% |
| qwen2 3B Q4_0 | 1.70 GiB | 3.09 B | CPU | 64 | tg128 | 39.12 ± 0.27 | 100% |
| qwen2 3B Q4_0 | 1.70 GiB | 3.09 B | CPU | 64 | tg256 | 39.31 ± 0.69 | 100% |
| qwen2 3B Q4_0 | 1.70 GiB | 3.09 B | CPU | 64 | tg512 | 40.52 ± 0.03 | 100% |
| qwen2 3B Q4_K_M | 1.79 GiB | 3.09 B | CPU | 64 | pp512 | 301.02 ± 1.74 | 147% |
| qwen2 3B Q4_K_M | 1.79 GiB | 3.09 B | CPU | 64 | pp1024 | 287.23 ± 0.20 | 101% |
| qwen2 3B Q4_K_M | 1.79 GiB | 3.09 B | CPU | 64 | pp2048 | 262.77 ± 1.81 | 101% |
| qwen2 3B Q4_K_M | 1.79 GiB | 3.09 B | CPU | 64 | tg128 | 18.80 ± 0.99 | 48% |
| qwen2 3B Q4_K_M | 1.79 GiB | 3.09 B | CPU | 64 | tg256 | 24.46 ± 3.04 | 83% |
| qwen2 3B Q4_K_M | 1.79 GiB | 3.09 B | CPU | 64 | tg512 | 36.32 ± 3.59 | 90% |
| qwen2 3B Q4_0_8_8 | 1.69 GiB | 3.09 B | CPU | 64 | pp512 | 271.71 ± 3.53 | 133% |
| qwen2 3B Q4_0_8_8 | 1.69 GiB | 3.09 B | CPU | 64 | pp1024 | 279.86 ± 45.63 | 100% |
| qwen2 3B Q4_0_8_8 | 1.69 GiB | 3.09 B | CPU | 64 | pp2048 | 320.77 ± 5.00 | 124% |
| qwen2 3B Q4_0_8_8 | 1.69 GiB | 3.09 B | CPU | 64 | tg128 | 43.51 ± 0.05 | 111% |
| qwen2 3B Q4_0_8_8 | 1.69 GiB | 3.09 B | CPU | 64 | tg256 | 43.35 ± 0.09 | 110% |
| qwen2 3B Q4_0_8_8 | 1.69 GiB | 3.09 B | CPU | 64 | tg512 | 42.60 ± 0.31 | 105% |
Q4_0_8_8 offers a nice bump to prompt processing and a small bump to text generation
</details>
</details>
## Which file should I choose?
<details>
<summary>Click here for details</summary>
A great write up with charts showing various performances is provided by Artefact2 [here](https://gist.github.com/Artefact2/b5f810600771265fc1e39442288e8ec9)
The first thing to figure out is how big a model you can run. To do this, you'll need to figure out how much RAM and/or VRAM you have.
If you want your model running as FAST as possible, you'll want to fit the whole thing on your GPU's VRAM. Aim for a quant with a file size 1-2GB smaller than your GPU's total VRAM.
If you want the absolute maximum quality, add both your system RAM and your GPU's VRAM together, then similarly grab a quant with a file size 1-2GB Smaller than that total.
Next, you'll need to decide if you want to use an 'I-quant' or a 'K-quant'.
If you don't want to think too much, grab one of the K-quants. These are in format 'QX_K_X', like Q5_K_M.
If you want to get more into the weeds, you can check out this extremely useful feature chart:
[llama.cpp feature matrix](https://github.com/ggerganov/llama.cpp/wiki/Feature-matrix)
But basically, if you're aiming for below Q4, and you're running cuBLAS (Nvidia) or rocBLAS (AMD), you should look towards the I-quants. These are in format IQX_X, like IQ3_M. These are newer and offer better performance for their size.
These I-quants can also be used on CPU and Apple Metal, but will be slower than their K-quant equivalent, so speed vs performance is a tradeoff you'll have to decide.
The I-quants are *not* compatible with Vulcan, which is also AMD, so if you have an AMD card double check if you're using the rocBLAS build or the Vulcan build. At the time of writing this, LM Studio has a preview with ROCm support, and other inference engines have specific builds for ROCm.
</details>
## Credits
Thank you kalomaze and Dampf for assistance in creating the imatrix calibration dataset.
Thank you ZeroWw for the inspiration to experiment with embed/output.
Want to support my work? Visit my ko-fi page here: https://ko-fi.com/bartowski

1
configuration.json Normal file
View File

@@ -0,0 +1 @@
{"framework": "pytorch", "task": "text-generation", "allow_remote": true}

3
cybertron-v4-qw7B-UNAMGS-IQ2_M.gguf Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:e7b08beda80560e677fc32db2e8ac94dfe03cec3e00df7a87be5124cecf42c98
size 2780343296

3
cybertron-v4-qw7B-UNAMGS-IQ3_M.gguf Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:c7695e464e9ce6e24c9b9dc1c88785866e7f6c16bb6d6c1d8c9a9492e1b21208
size 3574012928

3
cybertron-v4-qw7B-UNAMGS-IQ3_XS.gguf Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:2b8d85a22b2e15763029103ef824cc0152d49f245eec17896fffeb62c0e2eb13
size 3346256896

3
cybertron-v4-qw7B-UNAMGS-IQ4_XS.gguf Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:2b4e6b2a07cb76f33e1af0f63179de43f62e176d7925c2d15ad10c783812c979
size 4218473472

3
cybertron-v4-qw7B-UNAMGS-Q2_K.gguf Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:91f764fdc4f85d125e5ddbf399ed59753d05f03eb34b46616ad2ecdcf5e443ce
size 3015941120

3
cybertron-v4-qw7B-UNAMGS-Q2_K_L.gguf Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:2805e7ee821e07327be330d9b974a1c96108bfdd77da78bb982756f4ad609c2b
size 3548165120

3
cybertron-v4-qw7B-UNAMGS-Q3_K_L.gguf Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:f9d25c70ac8e8c999af1632fafab14e30f2652780c8e571a30365d6ca9d20ae3
size 4088460288

3
cybertron-v4-qw7B-UNAMGS-Q3_K_M.gguf Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:6203dea73e5f9ac0adfdf60e5f8dc3f3e675425ad85412b246856dbc8f0830f4
size 3808392192

3
cybertron-v4-qw7B-UNAMGS-Q3_K_S.gguf Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:5434f86d40ab47e8bf5ae8d3247456cfac1ecdda26d9057960788b5c7334394a
size 3492369408

3
cybertron-v4-qw7B-UNAMGS-Q3_K_XL.gguf Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:3080fccb3eafd1acb65b55b54af37382b930ed0505dbbd933d67a5f03ca7ab75
size 4565332992

3
cybertron-v4-qw7B-UNAMGS-Q4_0.gguf Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:f7debc01517ddd901647a711094286585b754cebcd5ab1e4c6af18db0547155c
size 4444122112

3
cybertron-v4-qw7B-UNAMGS-Q4_0_4_4.gguf Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:ef3c5cc195e68a07fbb9565981e3cc998fc0c6f5a837ac67b4385fbde8138a65
size 4431391744

3
cybertron-v4-qw7B-UNAMGS-Q4_0_4_8.gguf Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:8667bfcdae0322ca6ca6134253442fddc5c2b1ae7bd713e7b7f75a3d7ce54090
size 4431391744

3
cybertron-v4-qw7B-UNAMGS-Q4_0_8_8.gguf Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:587ec23c1546d102cbf3042be4ed8518515fb51489c46c54c26638cd460c1af8
size 4431391744

3
cybertron-v4-qw7B-UNAMGS-Q4_K_L.gguf Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:caa96d00a54fe071f38b7e93c7fe07881714034ea6fb0f0a3fcf8b026bd0a3ce
size 5087564800

3
cybertron-v4-qw7B-UNAMGS-Q4_K_M.gguf Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:decb4dc8cec094adcfae0ad379ba041667fceda8eb03161ba953d7101bd219c8
size 4683074560

3
cybertron-v4-qw7B-UNAMGS-Q4_K_S.gguf Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:26445cbd279ec1dee80994ce8ca734fff0038e0d5fd12587357908a725ff5f1c
size 4457769984

3
cybertron-v4-qw7B-UNAMGS-Q5_K_L.gguf Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:fe55df0ace3638b486bdf5a9948ee4f9d494f9d375722e3e99a033def0dadfb7
size 5781197824

3
cybertron-v4-qw7B-UNAMGS-Q5_K_M.gguf Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:f4e2144c6e02f68f7c2edcff1f2bff1de22b907dac7534911eacbe7204fbf1ff
size 5444832256

3
cybertron-v4-qw7B-UNAMGS-Q5_K_S.gguf Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:75bd54b4031ca2acc2ef62140f3eb99da69bb3014869e5fb40a22fd0950a6fb4
size 5315177472

3
cybertron-v4-qw7B-UNAMGS-Q6_K.gguf Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:82769fbebc751bc114944c34be5d3c0f5dea3661005819818c9d511730243fef
size 6254199808

3
cybertron-v4-qw7B-UNAMGS-Q6_K_L.gguf Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:eb6678793077d3c2522cfa38834147a65e4c6a0a0c8548e2b45ff8f08d555dbf
size 6518182912

3
cybertron-v4-qw7B-UNAMGS-Q8_0.gguf Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:950a77fc8f5f94ea0afcefad534427c1932be686fc0625d1020094b50a045406
size 8098526208

3
cybertron-v4-qw7B-UNAMGS-f16.gguf Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:c13da486b482d2d0b266f463b52f3bd6c4c32cfabc71549395350ee80ebfb269
size 15237853952

3
cybertron-v4-qw7B-UNAMGS.imatrix Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:359668d4895b52525f049b44af4a94991a066b490ce95d75c093c3647dfb2369
size 4536678