From 9e76151bc9e30c92f75bff4e4f436b92d2abe43a Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Fri, 24 Jul 2026 01:53:09 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: sh111111111111111/Qwen3-4B-Instruct-2507-BitClass2-GGUF Source: Original Platform --- .gitattributes | 40 +++++++++++++++ Qwen3-4B-Instruct-2507-Q3_K_S.gguf | 3 ++ Qwen3-4B-Instruct-2507-Q4_K_M.gguf | 3 ++ Qwen3-4B-Instruct-2507-Q5_K_M.gguf | 3 ++ Qwen3-4B-Instruct-2507-Q6_K.gguf | 3 ++ Qwen3-4B-Instruct-2507-Q8_0.gguf | 3 ++ README.md | 79 ++++++++++++++++++++++++++++++ 7 files changed, 134 insertions(+) create mode 100644 .gitattributes create mode 100644 Qwen3-4B-Instruct-2507-Q3_K_S.gguf create mode 100644 Qwen3-4B-Instruct-2507-Q4_K_M.gguf create mode 100644 Qwen3-4B-Instruct-2507-Q5_K_M.gguf create mode 100644 Qwen3-4B-Instruct-2507-Q6_K.gguf create mode 100644 Qwen3-4B-Instruct-2507-Q8_0.gguf create mode 100644 README.md diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..7209fa2 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,40 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +Qwen3-4B-Instruct-2507-Q4_K_M.gguf filter=lfs diff=lfs merge=lfs -text +Qwen3-4B-Instruct-2507-Q5_K_M.gguf filter=lfs diff=lfs merge=lfs -text +Qwen3-4B-Instruct-2507-Q8_0.gguf filter=lfs diff=lfs merge=lfs -text +Qwen3-4B-Instruct-2507-Q3_K_S.gguf filter=lfs diff=lfs merge=lfs -text +Qwen3-4B-Instruct-2507-Q6_K.gguf filter=lfs diff=lfs merge=lfs -text diff --git a/Qwen3-4B-Instruct-2507-Q3_K_S.gguf b/Qwen3-4B-Instruct-2507-Q3_K_S.gguf new file mode 100644 index 0000000..606cf8b --- /dev/null +++ b/Qwen3-4B-Instruct-2507-Q3_K_S.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:83ac55108915c8cead0e0eda9f7931b300b27d6c23e6ee8b26db87754521903c +size 1624344704 diff --git a/Qwen3-4B-Instruct-2507-Q4_K_M.gguf b/Qwen3-4B-Instruct-2507-Q4_K_M.gguf new file mode 100644 index 0000000..5f464ad --- /dev/null +++ b/Qwen3-4B-Instruct-2507-Q4_K_M.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:76e5ae3aebe3fd7bd8317bbf9280d294ec59c3fc840a7f47b0f07b9feb3c96ff +size 2346866304 diff --git a/Qwen3-4B-Instruct-2507-Q5_K_M.gguf b/Qwen3-4B-Instruct-2507-Q5_K_M.gguf new file mode 100644 index 0000000..196b19e --- /dev/null +++ b/Qwen3-4B-Instruct-2507-Q5_K_M.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:32726f25ada6a130cdc375165e21367b52e852cd204c8f6b344ee86d008c54c3 +size 2596988544 diff --git a/Qwen3-4B-Instruct-2507-Q6_K.gguf b/Qwen3-4B-Instruct-2507-Q6_K.gguf new file mode 100644 index 0000000..2d62b95 --- /dev/null +++ b/Qwen3-4B-Instruct-2507-Q6_K.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e567e777735b915f2357b34841482e3908edbe41657058b3bd85853dc1598d50 +size 2933728384 diff --git a/Qwen3-4B-Instruct-2507-Q8_0.gguf b/Qwen3-4B-Instruct-2507-Q8_0.gguf new file mode 100644 index 0000000..8a03c7d --- /dev/null +++ b/Qwen3-4B-Instruct-2507-Q8_0.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e9263a0c1433b00699206b03b427010ebdb3bccb8cf7ea1d9011a527a2e823be +size 4280403296 diff --git a/README.md b/README.md new file mode 100644 index 0000000..da5e937 --- /dev/null +++ b/README.md @@ -0,0 +1,79 @@ +--- +language: [en, zh] +license: apache-2.0 +library_name: gguf +base_model: Qwen/Qwen3-4B-Instruct-2507 +tags: [quantized, gguf, mixed-precision, bitclass, qwen3] +pipeline_tag: text-generation +--- + +# Qwen3-4B-Instruct-2507 — BitClass2 Mixed-Precision GGUF + +Mixed-precision GGUF quantizations of [Qwen3-4B-Instruct-2507](https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507) +using **Hessian-informed per-tensor bit allocation**. Each tensor group receives +the precision level that minimizes quality loss for its measured sensitivity — +more bits where they matter, fewer where they don't. + +## Available Quantizations + +| File | BPW | Size | PPL ↓ | tok/s | Use Case | +|---|---|---|---|---|---| +| [`Qwen3-4B-Instruct-2507-Q8_0.gguf`](./Qwen3-4B-Instruct-2507-Q8_0.gguf) | 8.5 | 4.28 GB | 2.651 | 11.4 | Near-lossless reference | +| [`Qwen3-4B-Instruct-2507-Q6_K.gguf`](./Qwen3-4B-Instruct-2507-Q6_K.gguf) | 5.8 | 2.93 GB | 2.888 | 13.6 | High quality, moderate size | +| [`Qwen3-4B-Instruct-2507-Q5_K_M.gguf`](./Qwen3-4B-Instruct-2507-Q5_K_M.gguf) | 5.2 | 2.60 GB | 2.971 | 14.3 | Balanced quality and size | +| [`Qwen3-4B-Instruct-2507-Q4_K_M.gguf`](./Qwen3-4B-Instruct-2507-Q4_K_M.gguf) | 4.7 | 2.35 GB | 2.978 | 14.1 | Best quality-to-size ratio | +| [`Qwen3-4B-Instruct-2507-Q3_K_S.gguf`](./Qwen3-4B-Instruct-2507-Q3_K_S.gguf) | 3.2 | 1.62 GB | 3.214 | 18.9 | Maximum compression | + +**Recommended:** Q4_K_M for the best quality-to-size ratio (PPL 2.978 at just 2.35 GB). +Q3_K_S for maximum compression. Q6_K for high quality. + +## How It Works + +Standard quantization applies one precision level uniformly across all tensors. +BitClass2 uses **Hessian-based sensitivity analysis** (H_diag = mean(X²) per layer) +to identify which tensors lose the most quality when quantized, then solves an +LP-optimal knapsack allocation: minimize Σ(sensitivity × quantization_error) +subject to total size ≤ target. Sensitive tensors get higher precision, +insensitive ones get lower precision, at the same total file size. + +Within each suffix group, the fractional BPW planner further varies types +per-layer using blended imatrix + Hessian scores, so late attention layers +(most sensitive) get higher precision than middle layers (least sensitive). + +## Key Sensitivity Findings (Qwen3-4B) + +- **Late attention layers (29-35) are most sensitive** — blk.34 k/v score 1.0 +- **down_proj is the most sensitive MLP tensor** — projects back to residual stream +- **gate_proj/up_proj are least sensitive** — safe to quantize aggressively +- **K > V for attention weight sensitivity** — k_proj averages 0.66 vs v_proj 0.50 + +## Usage + +```bash +# Download +huggingface-cli download sh111111111111111/Qwen3-4B-Instruct-2507-BitClass2-GGUF \ + Qwen3-4B-Instruct-2507-Q4_K_M.gguf --local-dir . + +# Chat with llama.cpp +llama-cli -m Qwen3-4B-Instruct-2507-Q4_K_M.gguf -cnv + +# Serve via API +llama-server -m Qwen3-4B-Instruct-2507-Q4_K_M.gguf --port 8080 + +# Ollama +ollama run hf.co/sh111111111111111/Qwen3-4B-Instruct-2507-BitClass2-GGUF:Qwen3-4B-Instruct-2507-Q4_K_M.gguf +``` + +## Benchmark Details + +All benchmarks run on NVIDIA GB10 ATOM (128GB unified memory, aarch64). +llama.cpp commit 406f4e3. PPL via `llama-perplexity` (2 chunks, 851 context). +tok/s via `llama-bench` (tg128, ngl=999). + +## Disclaimer + +Independent project. Not affiliated with or endorsed by Qwen, Unsloth, ByteShape, Bartowski, or llama.cpp. + +## License + +Apache 2.0, inherited from [Qwen3-4B-Instruct-2507](https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507).