From bfd8cb6775574e08eccc2111b019c57460625a1a Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Sat, 25 Jul 2026 08:04:11 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: ermiaazarkhalili/Granite-4.1-8B-SFT-Claude-Opus-Reasoning-Unsloth-GGUF Source: Original Platform --- .gitattributes | 41 +++++++++ README.md | 86 +++++++++++++++++++ ...ft-claude-opus-reasoning-unsloth.q2_k.gguf | 3 + ...-claude-opus-reasoning-unsloth.q3_k_m.gguf | 3 + ...-claude-opus-reasoning-unsloth.q4_k_m.gguf | 3 + ...-claude-opus-reasoning-unsloth.q5_k_m.gguf | 3 + ...ft-claude-opus-reasoning-unsloth.q6_k.gguf | 3 + ...ft-claude-opus-reasoning-unsloth.q8_0.gguf | 3 + 8 files changed, 145 insertions(+) create mode 100644 .gitattributes create mode 100644 README.md create mode 100644 granite-4.1-8b-sft-claude-opus-reasoning-unsloth.q2_k.gguf create mode 100644 granite-4.1-8b-sft-claude-opus-reasoning-unsloth.q3_k_m.gguf create mode 100644 granite-4.1-8b-sft-claude-opus-reasoning-unsloth.q4_k_m.gguf create mode 100644 granite-4.1-8b-sft-claude-opus-reasoning-unsloth.q5_k_m.gguf create mode 100644 granite-4.1-8b-sft-claude-opus-reasoning-unsloth.q6_k.gguf create mode 100644 granite-4.1-8b-sft-claude-opus-reasoning-unsloth.q8_0.gguf diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..0c40b2e --- /dev/null +++ b/.gitattributes @@ -0,0 +1,41 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +granite-4.1-8b-sft-claude-opus-reasoning-unsloth.q2_k.gguf filter=lfs diff=lfs merge=lfs -text +granite-4.1-8b-sft-claude-opus-reasoning-unsloth.q3_k_m.gguf filter=lfs diff=lfs merge=lfs -text +granite-4.1-8b-sft-claude-opus-reasoning-unsloth.q4_k_m.gguf filter=lfs diff=lfs merge=lfs -text +granite-4.1-8b-sft-claude-opus-reasoning-unsloth.q5_k_m.gguf filter=lfs diff=lfs merge=lfs -text +granite-4.1-8b-sft-claude-opus-reasoning-unsloth.q6_k.gguf filter=lfs diff=lfs merge=lfs -text +granite-4.1-8b-sft-claude-opus-reasoning-unsloth.q8_0.gguf filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..d12e623 --- /dev/null +++ b/README.md @@ -0,0 +1,86 @@ +--- +license: apache-2.0 +language: + - en +library_name: gguf +pipeline_tag: text-generation +tags: + - gguf + - llama.cpp + - quantized + - granite + - reasoning + - unsloth + - text-generation +base_model: ermiaazarkhalili/Granite-4.1-8B-SFT-Claude-Opus-Reasoning-Unsloth +datasets: + - ermiaazarkhalili/claude-reasoning-distillation +--- + +# Granite-4.1-8B-SFT-Claude-Opus-Reasoning-Unsloth-GGUF + +GGUF quantizations of [Granite-4.1-8B-SFT-Claude-Opus-Reasoning-Unsloth](https://huggingface.co/ermiaazarkhalili/Granite-4.1-8B-SFT-Claude-Opus-Reasoning-Unsloth) for **CPU and edge inference** with [llama.cpp](https://github.com/ggerganov/llama.cpp), [Ollama](https://ollama.com), LM Studio, and other GGUF runtimes. + +This model is a fine-tune of [Granite 4.1-8B](https://huggingface.co/ibm-granite/granite-4.1-8b) for **reasoning distillation (chain-of-thought)**, trained with [Unsloth](https://github.com/unslothai/unsloth) on [claude-reasoning-distillation](https://huggingface.co/datasets/ermiaazarkhalili/claude-reasoning-distillation). + +- **Full-precision model:** [Granite-4.1-8B-SFT-Claude-Opus-Reasoning-Unsloth](https://huggingface.co/ermiaazarkhalili/Granite-4.1-8B-SFT-Claude-Opus-Reasoning-Unsloth) +- **Base model:** [Granite 4.1-8B](https://huggingface.co/ibm-granite/granite-4.1-8b) +- **Parameters:** 8B + +## Available Quantizations + +| Quant | Size | Recommended Use | File | +|-------|------|-----------------|------| +| `Q2_K` | 3.41 GB | Smallest, lowest quality — quick tests / very constrained devices | [`granite-4.1-8b-sft-claude-opus-reasoning-unsloth.q2_k.gguf`](https://huggingface.co/ermiaazarkhalili/Granite-4.1-8B-SFT-Claude-Opus-Reasoning-Unsloth-GGUF/blob/main/granite-4.1-8b-sft-claude-opus-reasoning-unsloth.q2_k.gguf) | +| `Q3_K_M` | 4.35 GB | Small, acceptable quality | [`granite-4.1-8b-sft-claude-opus-reasoning-unsloth.q3_k_m.gguf`](https://huggingface.co/ermiaazarkhalili/Granite-4.1-8B-SFT-Claude-Opus-Reasoning-Unsloth-GGUF/blob/main/granite-4.1-8b-sft-claude-opus-reasoning-unsloth.q3_k_m.gguf) | +| `Q4_K_M` | 5.35 GB | Recommended — best size/quality balance | [`granite-4.1-8b-sft-claude-opus-reasoning-unsloth.q4_k_m.gguf`](https://huggingface.co/ermiaazarkhalili/Granite-4.1-8B-SFT-Claude-Opus-Reasoning-Unsloth-GGUF/blob/main/granite-4.1-8b-sft-claude-opus-reasoning-unsloth.q4_k_m.gguf) | +| `Q5_K_M` | 6.25 GB | High quality, larger | [`granite-4.1-8b-sft-claude-opus-reasoning-unsloth.q5_k_m.gguf`](https://huggingface.co/ermiaazarkhalili/Granite-4.1-8B-SFT-Claude-Opus-Reasoning-Unsloth-GGUF/blob/main/granite-4.1-8b-sft-claude-opus-reasoning-unsloth.q5_k_m.gguf) | +| `Q6_K` | 7.22 GB | Very high quality, near-fp16 | [`granite-4.1-8b-sft-claude-opus-reasoning-unsloth.q6_k.gguf`](https://huggingface.co/ermiaazarkhalili/Granite-4.1-8B-SFT-Claude-Opus-Reasoning-Unsloth-GGUF/blob/main/granite-4.1-8b-sft-claude-opus-reasoning-unsloth.q6_k.gguf) | +| `Q8_0` | 9.35 GB | Near-lossless, largest | [`granite-4.1-8b-sft-claude-opus-reasoning-unsloth.q8_0.gguf`](https://huggingface.co/ermiaazarkhalili/Granite-4.1-8B-SFT-Claude-Opus-Reasoning-Unsloth-GGUF/blob/main/granite-4.1-8b-sft-claude-opus-reasoning-unsloth.q8_0.gguf) | + +`Q4_K_M` is the recommended default for most users. + +## Usage + +### Download a single quant + +```bash +pip install -U "huggingface_hub[cli]" +hf download ermiaazarkhalili/Granite-4.1-8B-SFT-Claude-Opus-Reasoning-Unsloth-GGUF \ + --include "*q4_k_m*.gguf" --local-dir ./Granite-4.1-8B-SFT-Claude-Opus-Reasoning-Unsloth-GGUF +``` + +### llama.cpp + +```bash +# build: https://github.com/ggerganov/llama.cpp +./llama-cli -m ./Granite-4.1-8B-SFT-Claude-Opus-Reasoning-Unsloth-GGUF/granite-4.1-8b-sft-claude-opus-reasoning-unsloth.q2_k.gguf \ + -p "Solve step by step: What is the sum of the first 10 prime numbers?" -n 512 +``` + +### Ollama + +```bash +ollama run hf.co/ermiaazarkhalili/Granite-4.1-8B-SFT-Claude-Opus-Reasoning-Unsloth-GGUF:Q4_K_M "Solve step by step: What is the sum of the first 10 prime numbers?" +``` + +## Training Outcome + +| Metric | Value | +|--------|-------| +| SLURM Job ID | `38330897` | +| Runtime | 32m 49s | +| Final Training Loss | 0.7895 | +| Peak VRAM | 9.26 GB | +| GPU | H100 80GB HBM3 (MIG 3g.40gb) | + +## License + +APACHE-2.0 — see the [base model](https://huggingface.co/ibm-granite/granite-4.1-8b) for full terms. + +## Acknowledgments + +- [Unsloth](https://github.com/unslothai/unsloth) for 2x faster fine-tuning +- [llama.cpp](https://github.com/ggerganov/llama.cpp) for GGUF quantization +- Base model developers (ibm-granite) +- [Compute Canada / DRAC](https://alliancecan.ca/) for HPC resources diff --git a/granite-4.1-8b-sft-claude-opus-reasoning-unsloth.q2_k.gguf b/granite-4.1-8b-sft-claude-opus-reasoning-unsloth.q2_k.gguf new file mode 100644 index 0000000..c9f9d0f --- /dev/null +++ b/granite-4.1-8b-sft-claude-opus-reasoning-unsloth.q2_k.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:debdea4a3ca847e8131d5be83d23e31a02cd5cfef1922019115aa530c548791c +size 3412308512 diff --git a/granite-4.1-8b-sft-claude-opus-reasoning-unsloth.q3_k_m.gguf b/granite-4.1-8b-sft-claude-opus-reasoning-unsloth.q3_k_m.gguf new file mode 100644 index 0000000..cfe23fc --- /dev/null +++ b/granite-4.1-8b-sft-claude-opus-reasoning-unsloth.q3_k_m.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:96c881d600f8e99a3203736a9e1ee5531df744d08b51248d8385dcfaa304697d +size 4347048480 diff --git a/granite-4.1-8b-sft-claude-opus-reasoning-unsloth.q4_k_m.gguf b/granite-4.1-8b-sft-claude-opus-reasoning-unsloth.q4_k_m.gguf new file mode 100644 index 0000000..0a127df --- /dev/null +++ b/granite-4.1-8b-sft-claude-opus-reasoning-unsloth.q4_k_m.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8ae661481d778ef03f205768af734c1d8718984b11d027793623ccfa932b6f8d +size 5347914272 diff --git a/granite-4.1-8b-sft-claude-opus-reasoning-unsloth.q5_k_m.gguf b/granite-4.1-8b-sft-claude-opus-reasoning-unsloth.q5_k_m.gguf new file mode 100644 index 0000000..878f74e --- /dev/null +++ b/granite-4.1-8b-sft-claude-opus-reasoning-unsloth.q5_k_m.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4382263c2e3d7cebff6396bfd0bd3d0e202884706d7545f000b599fb3d339992 +size 6253883936 diff --git a/granite-4.1-8b-sft-claude-opus-reasoning-unsloth.q6_k.gguf b/granite-4.1-8b-sft-claude-opus-reasoning-unsloth.q6_k.gguf new file mode 100644 index 0000000..73ce369 --- /dev/null +++ b/granite-4.1-8b-sft-claude-opus-reasoning-unsloth.q6_k.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e5f3d8c6146664fde011818c53d95aa7e94efb1ff3e22f169b8f822c45e7d980 +size 7216476704 diff --git a/granite-4.1-8b-sft-claude-opus-reasoning-unsloth.q8_0.gguf b/granite-4.1-8b-sft-claude-opus-reasoning-unsloth.q8_0.gguf new file mode 100644 index 0000000..2539c4d --- /dev/null +++ b/granite-4.1-8b-sft-claude-opus-reasoning-unsloth.q8_0.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2121d5ce820ad591277ec8aba7d90f3f1e1a9999c1fa9ef61d00d469ada30a18 +size 9345610272