From 59154da66d830a12c175fd0fcc4af8bba97464ca Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Mon, 25 May 2026 19:10:16 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: matrixportalx/Buttocks-7B-v2.1-GGUF Source: Original Platform --- .gitattributes | 48 +++++++++++ README.md | 157 +++++++++++++++++++++++++++++++++++ buttocks-7b-v2.1-f16.gguf | 3 + buttocks-7b-v2.1-q2_k.gguf | 3 + buttocks-7b-v2.1-q3_k_l.gguf | 3 + buttocks-7b-v2.1-q3_k_m.gguf | 3 + buttocks-7b-v2.1-q3_k_s.gguf | 3 + buttocks-7b-v2.1-q4_0.gguf | 3 + buttocks-7b-v2.1-q4_k_m.gguf | 3 + buttocks-7b-v2.1-q4_k_s.gguf | 3 + buttocks-7b-v2.1-q5_0.gguf | 3 + buttocks-7b-v2.1-q5_k_m.gguf | 3 + buttocks-7b-v2.1-q5_k_s.gguf | 3 + buttocks-7b-v2.1-q6_k.gguf | 3 + buttocks-7b-v2.1-q8_0.gguf | 3 + 15 files changed, 244 insertions(+) create mode 100644 .gitattributes create mode 100644 README.md create mode 100644 buttocks-7b-v2.1-f16.gguf create mode 100644 buttocks-7b-v2.1-q2_k.gguf create mode 100644 buttocks-7b-v2.1-q3_k_l.gguf create mode 100644 buttocks-7b-v2.1-q3_k_m.gguf create mode 100644 buttocks-7b-v2.1-q3_k_s.gguf create mode 100644 buttocks-7b-v2.1-q4_0.gguf create mode 100644 buttocks-7b-v2.1-q4_k_m.gguf create mode 100644 buttocks-7b-v2.1-q4_k_s.gguf create mode 100644 buttocks-7b-v2.1-q5_0.gguf create mode 100644 buttocks-7b-v2.1-q5_k_m.gguf create mode 100644 buttocks-7b-v2.1-q5_k_s.gguf create mode 100644 buttocks-7b-v2.1-q6_k.gguf create mode 100644 buttocks-7b-v2.1-q8_0.gguf diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..45f699d --- /dev/null +++ b/.gitattributes @@ -0,0 +1,48 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +buttocks-7b-v2.1-q2_k.gguf filter=lfs diff=lfs merge=lfs -text +buttocks-7b-v2.1-q3_k_s.gguf filter=lfs diff=lfs merge=lfs -text +buttocks-7b-v2.1-q3_k_m.gguf filter=lfs diff=lfs merge=lfs -text +buttocks-7b-v2.1-q3_k_l.gguf filter=lfs diff=lfs merge=lfs -text +buttocks-7b-v2.1-q4_0.gguf filter=lfs diff=lfs merge=lfs -text +buttocks-7b-v2.1-q8_0.gguf filter=lfs diff=lfs merge=lfs -text +buttocks-7b-v2.1-q4_k_s.gguf filter=lfs diff=lfs merge=lfs -text +buttocks-7b-v2.1-q4_k_m.gguf filter=lfs diff=lfs merge=lfs -text +buttocks-7b-v2.1-q5_0.gguf filter=lfs diff=lfs merge=lfs -text +buttocks-7b-v2.1-q5_k_s.gguf filter=lfs diff=lfs merge=lfs -text +buttocks-7b-v2.1-q5_k_m.gguf filter=lfs diff=lfs merge=lfs -text +buttocks-7b-v2.1-q6_k.gguf filter=lfs diff=lfs merge=lfs -text +buttocks-7b-v2.1-f16.gguf filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..604eba0 --- /dev/null +++ b/README.md @@ -0,0 +1,157 @@ +--- +base_model: TeeZee/Buttocks-7B-v2.1 +library_name: transformers +tags: +- mergekit +- merge +- llama-cpp +- matrixportal +--- + +# matrixportal/Buttocks-7B-v2.1-GGUF + This model was converted to GGUF format from [`TeeZee/Buttocks-7B-v2.1`](https://huggingface.co/TeeZee/Buttocks-7B-v2.1) using llama.cpp via the ggml.ai's [all-gguf-same-where](https://huggingface.co/spaces/matrixportal/all-gguf-same-where) space. +Refer to the [original model card](https://huggingface.co/TeeZee/Buttocks-7B-v2.1) for more details on the model. + +## ✅ Quantized Models Download List + +### 🔍 Recommended Quantizations +| 🚀 Download | 🔢 Type | 📝 Notes | +|:---------|:-----|:------| +| [Download](https://huggingface.co/matrixportal/Buttocks-7B-v2.1-GGUF/resolve/main/buttocks-7b-v2.1-q2_k.gguf) | ![Q2_K](https://img.shields.io/badge/Q2_K-1A73E8) | Basic quantization | +| [Download](https://huggingface.co/matrixportal/Buttocks-7B-v2.1-GGUF/resolve/main/buttocks-7b-v2.1-q3_k_s.gguf) | ![Q3_K_S](https://img.shields.io/badge/Q3_K_S-34A853) | Small size | +| [Download](https://huggingface.co/matrixportal/Buttocks-7B-v2.1-GGUF/resolve/main/buttocks-7b-v2.1-q3_k_m.gguf) | ![Q3_K_M](https://img.shields.io/badge/Q3_K_M-FBBC05) | Balanced quality | +| [Download](https://huggingface.co/matrixportal/Buttocks-7B-v2.1-GGUF/resolve/main/buttocks-7b-v2.1-q3_k_l.gguf) | ![Q3_K_L](https://img.shields.io/badge/Q3_K_L-4285F4) | Better quality | +| [Download](https://huggingface.co/matrixportal/Buttocks-7B-v2.1-GGUF/resolve/main/buttocks-7b-v2.1-q4_0.gguf) | ![Q4_0](https://img.shields.io/badge/Q4_0-EA4335) | Fast on ARM | +| [Download](https://huggingface.co/matrixportal/Buttocks-7B-v2.1-GGUF/resolve/main/buttocks-7b-v2.1-q4_k_s.gguf) | ![Q4_K_S](https://img.shields.io/badge/Q4_K_S-673AB7) | Fast, recommended | +| [Download](https://huggingface.co/matrixportal/Buttocks-7B-v2.1-GGUF/resolve/main/buttocks-7b-v2.1-q4_k_m.gguf) | ![Q4_K_M](https://img.shields.io/badge/Q4_K_M-673AB7) ⭐ | Best balance | +| [Download](https://huggingface.co/matrixportal/Buttocks-7B-v2.1-GGUF/resolve/main/buttocks-7b-v2.1-q5_0.gguf) | ![Q5_0](https://img.shields.io/badge/Q5_0-FF6D01) | Good quality | +| [Download](https://huggingface.co/matrixportal/Buttocks-7B-v2.1-GGUF/resolve/main/buttocks-7b-v2.1-q5_k_s.gguf) | ![Q5_K_S](https://img.shields.io/badge/Q5_K_S-0F9D58) | Balanced | +| [Download](https://huggingface.co/matrixportal/Buttocks-7B-v2.1-GGUF/resolve/main/buttocks-7b-v2.1-q5_k_m.gguf) | ![Q5_K_M](https://img.shields.io/badge/Q5_K_M-0F9D58) | High quality | +| [Download](https://huggingface.co/matrixportal/Buttocks-7B-v2.1-GGUF/resolve/main/buttocks-7b-v2.1-q6_k.gguf) | ![Q6_K](https://img.shields.io/badge/Q6_K-4285F4) 🏆 | Very good quality | +| [Download](https://huggingface.co/matrixportal/Buttocks-7B-v2.1-GGUF/resolve/main/buttocks-7b-v2.1-q8_0.gguf) | ![Q8_0](https://img.shields.io/badge/Q8_0-EA4335) ⚡ | Fast, best quality | +| [Download](https://huggingface.co/matrixportal/Buttocks-7B-v2.1-GGUF/resolve/main/buttocks-7b-v2.1-f16.gguf) | ![F16](https://img.shields.io/badge/F16-000000) | Maximum accuracy | + +💡 **Tip:** Use `F16` for maximum precision when quality is critical + +# GGUF Model Quantization & Usage Guide with llama.cpp + +## What is GGUF and Quantization? + +**GGUF** (GPT-Generated Unified Format) is an efficient model file format developed by the `llama.cpp` team that: +- Supports multiple quantization levels +- Works cross-platform +- Enables fast loading and inference + +**Quantization** converts model weights to lower precision data types (e.g., 4-bit integers instead of 32-bit floats) to: +- Reduce model size +- Decrease memory usage +- Speed up inference +- (With minor accuracy trade-offs) + +## Step-by-Step Guide + +### 1. Prerequisites + +```bash +# System updates +sudo apt update && sudo apt upgrade -y + +# Dependencies +sudo apt install -y build-essential cmake python3-pip + +# Clone and build llama.cpp +git clone https://github.com/ggerganov/llama.cpp +cd llama.cpp +make -j4 +``` + +### 2. Using Quantized Models from Hugging Face + +My automated quantization script produces models in this format: +``` +https://huggingface.co/matrixportal/Buttocks-7B-v2.1-GGUF/resolve/main/buttocks-7b-v2.1-q4_k_m.gguf +``` + +Download your quantized model directly: + +```bash +wget https://huggingface.co/matrixportal/Buttocks-7B-v2.1-GGUF/resolve/main/buttocks-7b-v2.1-q4_k_m.gguf +``` + +### 3. Running the Quantized Model + +Basic usage: +```bash +./main -m buttocks-7b-v2.1-q4_k_m.gguf -p "Your prompt here" -n 128 +``` + +Example with a creative writing prompt: +```bash +./main -m buttocks-7b-v2.1-q4_k_m.gguf -p "[INST] Write a short poem about AI quantization in the style of Shakespeare [/INST]" -n 256 -c 2048 -t 8 --temp 0.7 +``` + +Advanced parameters: +```bash +./main -m buttocks-7b-v2.1-q4_k_m.gguf -p "Question: What is the GGUF format? +Answer:" -n 256 -c 2048 -t 8 --temp 0.7 --top-k 40 --top-p 0.9 +``` + +### 4. Python Integration + +Install the Python package: +```bash +pip install llama-cpp-python +``` + +Example script: +```python +from llama_cpp import Llama + +# Initialize the model +llm = Llama( + model_path="buttocks-7b-v2.1-q4_k_m.gguf", + n_ctx=2048, + n_threads=8 +) + +# Run inference +response = llm( + "[INST] Explain GGUF quantization to a beginner [/INST]", + max_tokens=256, + temperature=0.7, + top_p=0.9 +) + +print(response["choices"][0]["text"]) +``` + +## Performance Tips + +1. **Hardware Utilization**: + - Set thread count with `-t` (typically CPU core count) + - Compile with CUDA/OpenCL for GPU support + +2. **Memory Optimization**: + - Lower quantization (like q4_k_m) uses less RAM + - Adjust context size with `-c` parameter + +3. **Speed/Accuracy Balance**: + - Higher bit quantization is slower but more accurate + - Reduce randomness with `--temp 0` for consistent results + +## FAQ + +**Q: What quantization levels are available?** +A: Common options include q4_0, q4_k_m, q5_0, q5_k_m, q8_0 (my script uses q4_k_m by default) + +**Q: How much performance loss occurs with q4_k_m?** +A: Typically 2-5% accuracy reduction but 4x smaller size + +**Q: How to enable GPU support?** +A: Build with `make LLAMA_CUBLAS=1` for NVIDIA GPUs + +## Useful Resources + +1. [llama.cpp GitHub](https://github.com/ggerganov/llama.cpp) +2. [GGUF Format Specs](https://github.com/ggerganov/ggml/blob/master/docs/gguf.md) +3. [Hugging Face Model Hub](https://huggingface.co/models) diff --git a/buttocks-7b-v2.1-f16.gguf b/buttocks-7b-v2.1-f16.gguf new file mode 100644 index 0000000..f081abe --- /dev/null +++ b/buttocks-7b-v2.1-f16.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29bc27833e5aefc1a8c192a2df5b222bdc9dc820b8f0a1945a03fa6128d34e0b +size 14484732576 diff --git a/buttocks-7b-v2.1-q2_k.gguf b/buttocks-7b-v2.1-q2_k.gguf new file mode 100644 index 0000000..7d962a7 --- /dev/null +++ b/buttocks-7b-v2.1-q2_k.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:21d911009370e04e2cd413ee2a1e5784be740e7e2c8d75eee14451fd85d6bc47 +size 2719242912 diff --git a/buttocks-7b-v2.1-q3_k_l.gguf b/buttocks-7b-v2.1-q3_k_l.gguf new file mode 100644 index 0000000..94bf5ac --- /dev/null +++ b/buttocks-7b-v2.1-q3_k_l.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ba8ec0f9757adea3411fd545aa98bb7df05a9b3aa07900987e613885f289d17c +size 3822025376 diff --git a/buttocks-7b-v2.1-q3_k_m.gguf b/buttocks-7b-v2.1-q3_k_m.gguf new file mode 100644 index 0000000..5501c66 --- /dev/null +++ b/buttocks-7b-v2.1-q3_k_m.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3a2707b8187510d04ab514df1aea1281eae28236c2f1072b6087d4fa1d8cde70 +size 3518986912 diff --git a/buttocks-7b-v2.1-q3_k_s.gguf b/buttocks-7b-v2.1-q3_k_s.gguf new file mode 100644 index 0000000..6ca3ce2 --- /dev/null +++ b/buttocks-7b-v2.1-q3_k_s.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:af17d999076a7b0508bf66bbeba18aab6145872d4f06ad192b6051af982ae174 +size 3164568224 diff --git a/buttocks-7b-v2.1-q4_0.gguf b/buttocks-7b-v2.1-q4_0.gguf new file mode 100644 index 0000000..22674f5 --- /dev/null +++ b/buttocks-7b-v2.1-q4_0.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5dfd0ba251c90bbcff625e99cae87ab888d8dd3c119cf0b5da188f51d4fee32b +size 4108917408 diff --git a/buttocks-7b-v2.1-q4_k_m.gguf b/buttocks-7b-v2.1-q4_k_m.gguf new file mode 100644 index 0000000..38c1193 --- /dev/null +++ b/buttocks-7b-v2.1-q4_k_m.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f096c2150fb5cc8c63bfc09fa51f05ae6d42b24f8682e0c0f64e62ff753a5ead +size 4368439968 diff --git a/buttocks-7b-v2.1-q4_k_s.gguf b/buttocks-7b-v2.1-q4_k_s.gguf new file mode 100644 index 0000000..4a58d28 --- /dev/null +++ b/buttocks-7b-v2.1-q4_k_s.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c8484caa5c08e2a20636af59d0e39598231412e948e8dd6a3c3f0e2f17edfb5a +size 4140374688 diff --git a/buttocks-7b-v2.1-q5_0.gguf b/buttocks-7b-v2.1-q5_0.gguf new file mode 100644 index 0000000..13486c5 --- /dev/null +++ b/buttocks-7b-v2.1-q5_0.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:df9b25faf84584b3e0863a8871c3f124dc12246a51d547927c1e1a6987819539 +size 4997716640 diff --git a/buttocks-7b-v2.1-q5_k_m.gguf b/buttocks-7b-v2.1-q5_k_m.gguf new file mode 100644 index 0000000..51dbddf --- /dev/null +++ b/buttocks-7b-v2.1-q5_k_m.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3b60ead22f7f890f09b2d6173aa82a173805a8883b5e2257814a97b3698085c2 +size 5131410080 diff --git a/buttocks-7b-v2.1-q5_k_s.gguf b/buttocks-7b-v2.1-q5_k_s.gguf new file mode 100644 index 0000000..15c5648 --- /dev/null +++ b/buttocks-7b-v2.1-q5_k_s.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:620421e5d3d186b7e2ae9e6a79467eed64ecb10794fe9700c1ec8dcc1e0e8e7e +size 4997716640 diff --git a/buttocks-7b-v2.1-q6_k.gguf b/buttocks-7b-v2.1-q6_k.gguf new file mode 100644 index 0000000..7ba0e27 --- /dev/null +++ b/buttocks-7b-v2.1-q6_k.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6e2ab58c47673b8617641865c33ea11b872a34a923916268544350166961a060 +size 5942065824 diff --git a/buttocks-7b-v2.1-q8_0.gguf b/buttocks-7b-v2.1-q8_0.gguf new file mode 100644 index 0000000..8d17c82 --- /dev/null +++ b/buttocks-7b-v2.1-q8_0.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d0f601830cbfd7f196159181692d648ba087ff69173ec48c249dc59e10254043 +size 7695858336