From 864e32062726305010abb445a2b2c66411c83ed3 Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Sun, 21 Jun 2026 06:27:16 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: SolusOps/Lizzy-7B-GGUF Source: Original Platform --- .gitattributes | 41 ++++++++++++++ README.md | 127 +++++++++++++++++++++++++++++++++++++++++++ lizzy-7b-Q3_K_M.gguf | 3 + lizzy-7b-Q4_K_M.gguf | 3 + lizzy-7b-Q5_K_M.gguf | 3 + lizzy-7b-Q6_K.gguf | 3 + lizzy-7b-Q8_0.gguf | 3 + lizzy-7b-f16.gguf | 3 + 8 files changed, 186 insertions(+) create mode 100644 .gitattributes create mode 100644 README.md create mode 100644 lizzy-7b-Q3_K_M.gguf create mode 100644 lizzy-7b-Q4_K_M.gguf create mode 100644 lizzy-7b-Q5_K_M.gguf create mode 100644 lizzy-7b-Q6_K.gguf create mode 100644 lizzy-7b-Q8_0.gguf create mode 100644 lizzy-7b-f16.gguf diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..59006c4 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,41 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +lizzy-7b-f16.gguf filter=lfs diff=lfs merge=lfs -text +lizzy-7b-Q4_K_M.gguf filter=lfs diff=lfs merge=lfs -text +lizzy-7b-Q8_0.gguf filter=lfs diff=lfs merge=lfs -text +lizzy-7b-Q3_K_M.gguf filter=lfs diff=lfs merge=lfs -text +lizzy-7b-Q5_K_M.gguf filter=lfs diff=lfs merge=lfs -text +lizzy-7b-Q6_K.gguf filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..f9295c2 --- /dev/null +++ b/README.md @@ -0,0 +1,127 @@ +--- +license: apache-2.0 +base_model: flwrlabs/Lizzy-7B +tags: + - llama-cpp + - gguf + - olmo2 + - quantized + - uk-english + - agentic + - function-calling + - lizzy-7B +language: + - en +pipeline_tag: text-generation +--- + +# Lizzy-7B GGUF Quants + +> 🚨 **Update:** Flower Labs has officially released their native GGUF quants. +> I highly recommend transitioning to their repository for the most stable inference and the corrected 32k context window: **[flwrlabs/Lizzy-7B-GGUF](https://huggingface.co/flwrlabs/Lizzy-7B-GGUF)**. +> +> *Note: During testing, I came across a bug with rope/context length issue, which has been patched in the official release. Thanks to the 250+ community members who tested this early build!* + +**Quantized by [SolusOps](https://huggingface.co/SolusOps)** + +**Original model:** [FlowerLabs/Lizzy-7B](https://huggingface.co/flwrlabs/Lizzy-7B) + +**Official Quants:** [flwrlabs/Lizzy-7B-GGUF](https://huggingface.co/flwrlabs/Lizzy-7B-GGUF) + +## About This Repo + +This repository provides llama.cpp-compatible GGUF quants of **Lizzy-7B**, a UK-centric 7B language model built by [Flower Labs](https://flower.ai). +Refer to the [original model card](https://huggingface.co/flwrlabs/Lizzy-7B) for more details on the model. + +## Available Quants + +| File | Quant | Size | Use Case | +|---|---|---|---| +| `Lizzy-7B-f16.gguf` | F16 | ~14.6 GB | needs 20GB+ VRAM or CPU offload. | +| `Lizzy-7B-Q8_0.gguf` | Q8_0 | ~7.7 GB | **Recommended** fits 12GB VRAM with excellent context headroom. | +| `Lizzy-7B-Q6_K.gguf` | Q6_K | ~5.9 GB | for 10GB–12GB GPUs looking to maximize context size. | +| `Lizzy-7B-Q5_K_M.gguf` | Q5_K_M | ~5.1 GB | 8GB VRAM | +| `Lizzy-7B-Q4_K_M.gguf` | Q4_K_M | ~4.1 GB | 6GB–8GB GPUs. | +| `Lizzy-7B-Q3_K_M.gguf` | Q3_K_M | ~3.5 GB | edge devices, 4GB GPUs, or older laptops. | + +## Hardware Tested + +| Hardware | Quant | n_ctx | Speed | +|---|---|---|---| +| RTX 3060 12GB | Q8_0 | 8192 | ~23 tok/s | +| RTX 3060 12GB | F16 | 4096 | Slower (VRAM overflow to RAM) | + +## Conversion Notes + +### 1. Architecture: OLMo 2 Post-Norm Tensor Mapping + +Lizzy-7B uses a Post-Norm variant of OLMo 2. +The standard convert_hf_to_gguf.py script does not recognise Flower Labs tensor naming conventions (post_attn_norm, post_mlp_norm) +and will fail or silently produce a broken file. +The fix was to register a LizzyForCausalLM model class in the llama.cpp conversion script, +subclassing Olmo2Model and overriding modify_tensors() to remap the four divergent tensor names: + +``` +python@ModelBase.register("LizzyForCausalLM") +class LizzyModel(Olmo2Model): + def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: + + # 1. Lizzy: post_attn_norm -> llama.cpp: post_attention_norm + if name.endswith(".post_attn_norm.weight"): + yield (f"blk.{bid}.post_attention_norm.weight", data_torch) + return + + # 2. Lizzy: post_mlp_norm -> llama.cpp: post_ffw_norm + if name.endswith(".post_mlp_norm.weight"): + yield (f"blk.{bid}.post_ffw_norm.weight", data_torch) + return + + # 3. QK-Norms these mapped correctly via standard paths + if name.endswith(".q_norm.weight"): + yield (self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_Q_NORM, bid), data_torch) + return + + if name.endswith(".k_norm.weight"): + yield (self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_K_NORM, bid), data_torch) + return + + # 4. All other tensors — pass through normally + yield from super().modify_tensors(data_torch, name, bid) + +``` + +No weights were altered. Only the tensor name metadata was remapped. + +### 2. RoPE Scaling Factor Correction + +During conversion, the script raised this warning: + +``` +The explicitly set RoPE scaling factor (config.rope_parameters['factor'] = 8.0) +does not match the ratio implicitly set by other parameters +(implicit factor = max_position_embeddings / original_max_position_embeddings = 4.0). +Using the explicit factor (8.0) in YaRN. This may cause unexpected behaviour. +``` + +The implicit factor (4.0) is mathematically derived from the model's own position embedding settings. The explicit `8.0` in the upstream config appears to be an authoring error. To produce a consistent and correctly-behaving GGUF, **the factor was corrected from `8.0` to `4.0`** in `config.json` before conversion. + +This means the effective context window for these GGUFs reflects the 4.0× YaRN scaling, not 8.0×. If Flower Labs corrects the upstream config, a re-conversion would be straightforward. + +## License + +The original Lizzy-7B model is released under **Apache 2.0** by Flower Labs. These quants inherit that license. + +## Links + +- [Original Model: FlowerLabs/Lizzy-7B](https://huggingface.co/flwrlabs/Lizzy-7B) +- [Flower Labs](https://flower.ai) +- [llama.cpp](https://github.com/ggerganov/llama.cpp) + +### About Me + +This GGUF port was completed by **Anshuman Singh**. + +* **GitHub:** [github.com/SolusOps](https://github.com/solusops) +* **LinkedIn:** [linkedin.com/in/anshumansingh2023](https://www.linkedin.com/in/anshumansingh2023/) + +If this port helped your local deployment, feel free to connect! diff --git a/lizzy-7b-Q3_K_M.gguf b/lizzy-7b-Q3_K_M.gguf new file mode 100644 index 0000000..f48c364 --- /dev/null +++ b/lizzy-7b-Q3_K_M.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1745e2f313d9b7b07ab85ccd98c03f2874c09af1df13ea63c298689f18a3ba03 +size 3651463136 diff --git a/lizzy-7b-Q4_K_M.gguf b/lizzy-7b-Q4_K_M.gguf new file mode 100644 index 0000000..0185131 --- /dev/null +++ b/lizzy-7b-Q4_K_M.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c1f94844a94042927b37bf96e95a0eff31f87f5918aca11c92c6aaf098d478ec +size 4471605920 diff --git a/lizzy-7b-Q5_K_M.gguf b/lizzy-7b-Q5_K_M.gguf new file mode 100644 index 0000000..5423d85 --- /dev/null +++ b/lizzy-7b-Q5_K_M.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5c4f3050b55432a3ebcf6e114968ee889525c30158d6e0c13471718cc116af4 +size 5208716960 diff --git a/lizzy-7b-Q6_K.gguf b/lizzy-7b-Q6_K.gguf new file mode 100644 index 0000000..dcf49b9 --- /dev/null +++ b/lizzy-7b-Q6_K.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4b42f3c4afcc3e736f4e2afe619b5c344ebb605d59066c7af09f3e4df9f1ba42 +size 5991897440 diff --git a/lizzy-7b-Q8_0.gguf b/lizzy-7b-Q8_0.gguf new file mode 100644 index 0000000..f19be3e --- /dev/null +++ b/lizzy-7b-Q8_0.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cb267e7f5e8f4ad0c2827fa91110d5feea2e1d7998defd781b460d9eb09bafd5 +size 7759256544 diff --git a/lizzy-7b-f16.gguf b/lizzy-7b-f16.gguf new file mode 100644 index 0000000..2191c22 --- /dev/null +++ b/lizzy-7b-f16.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e4a97ef754702166948a1be872b1e2f5870898a850f061b4ada4bba10677d717 +size 14600646624