commit 1e72a0868cae5c0d0ed350e4817bb3320b4af159 Author: ModelHub XC Date: Sun Sep 20 11:30:16 2026 +0800 初始化项目，由ModelHub XC社区提供模型 Model: matrixportalx/txgemma-9b-chat-GGUF Source: Original Platform diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..2206dd7 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,48 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +txgemma-9b-chat-q2_k.gguf filter=lfs diff=lfs merge=lfs -text +txgemma-9b-chat-q3_k_s.gguf filter=lfs diff=lfs merge=lfs -text +txgemma-9b-chat-q3_k_m.gguf filter=lfs diff=lfs merge=lfs -text +txgemma-9b-chat-q3_k_l.gguf filter=lfs diff=lfs merge=lfs -text +txgemma-9b-chat-q4_0.gguf filter=lfs diff=lfs merge=lfs -text +txgemma-9b-chat-q4_k_m.gguf filter=lfs diff=lfs merge=lfs -text +txgemma-9b-chat-q5_0.gguf filter=lfs diff=lfs merge=lfs -text +txgemma-9b-chat-q5_k_s.gguf filter=lfs diff=lfs merge=lfs -text +txgemma-9b-chat-q5_k_m.gguf filter=lfs diff=lfs merge=lfs -text +txgemma-9b-chat-q8_0.gguf filter=lfs diff=lfs merge=lfs -text +txgemma-9b-chat-f16.gguf filter=lfs diff=lfs merge=lfs -text +txgemma-9b-chat-q6_k.gguf filter=lfs diff=lfs merge=lfs -text +txgemma-9b-chat-q4_k_s.gguf filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..e971deb --- /dev/null +++ b/README.md @@ -0,0 +1,174 @@ +--- +base_model: google/txgemma-9b-chat +language: +- en +library_name: transformers +license: other +license_name: health-ai-developer-foundations +license_link: https://developers.google.com/health-ai-developer-foundations/terms +pipeline_tag: text-generation +tags: +- therapeutics +- drug-development +- llama-cpp +- matrixportal +extra_gated_heading: Access TxGemma on Hugging Face +extra_gated_prompt: To access TxGemma on Hugging Face, you're required to review and + agree to [Health AI Developer Foundation's terms of use](https://developers.google.com/health-ai-developer-foundations/terms). + To do this, please ensure you're logged in to Hugging Face and click below. Requests + are processed immediately. +extra_gated_button_content: Acknowledge license +--- + +# matrixportal/txgemma-9b-chat-GGUF + This model was converted to GGUF format from [`google/txgemma-9b-chat`](https://huggingface.co/google/txgemma-9b-chat) using llama.cpp via the ggml.ai's [all-gguf-same-where](https://huggingface.co/spaces/matrixportal/all-gguf-same-where) space. +Refer to the [original model card](https://huggingface.co/google/txgemma-9b-chat) for more details on the model. + +## ✅ Quantized Models Download List + +### 🔍 Recommended Quantizations +- **✨ General CPU Use:** [`Q4_K_M`](https://huggingface.co/matrixportal/txgemma-9b-chat-GGUF/resolve/main/txgemma-9b-chat-q4_k_m.gguf) (Best balance of speed/quality) +- **📱 ARM Devices:** [`Q4_0`](https://huggingface.co/matrixportal/txgemma-9b-chat-GGUF/resolve/main/txgemma-9b-chat-q4_0.gguf) (Optimized for ARM CPUs) +- **🏆 Maximum Quality:** [`Q8_0`](https://huggingface.co/matrixportal/txgemma-9b-chat-GGUF/resolve/main/txgemma-9b-chat-q8_0.gguf) (Near-original quality) + +### 📦 Full Quantization Options +| 🚀 Download | 🔢 Type | 📝 Notes | +|:---------|:-----|:------| +| [Download](https://huggingface.co/matrixportal/txgemma-9b-chat-GGUF/resolve/main/txgemma-9b-chat-q2_k.gguf) | ![Q2_K](https://img.shields.io/badge/Q2_K-1A73E8) | Basic quantization | +| [Download](https://huggingface.co/matrixportal/txgemma-9b-chat-GGUF/resolve/main/txgemma-9b-chat-q3_k_s.gguf) | ![Q3_K_S](https://img.shields.io/badge/Q3_K_S-34A853) | Small size | +| [Download](https://huggingface.co/matrixportal/txgemma-9b-chat-GGUF/resolve/main/txgemma-9b-chat-q3_k_m.gguf) | ![Q3_K_M](https://img.shields.io/badge/Q3_K_M-FBBC05) | Balanced quality | +| [Download](https://huggingface.co/matrixportal/txgemma-9b-chat-GGUF/resolve/main/txgemma-9b-chat-q3_k_l.gguf) | ![Q3_K_L](https://img.shields.io/badge/Q3_K_L-4285F4) | Better quality | +| [Download](https://huggingface.co/matrixportal/txgemma-9b-chat-GGUF/resolve/main/txgemma-9b-chat-q4_0.gguf) | ![Q4_0](https://img.shields.io/badge/Q4_0-EA4335) | Fast on ARM | +| [Download](https://huggingface.co/matrixportal/txgemma-9b-chat-GGUF/resolve/main/txgemma-9b-chat-q4_k_s.gguf) | ![Q4_K_S](https://img.shields.io/badge/Q4_K_S-673AB7) | Fast, recommended | +| [Download](https://huggingface.co/matrixportal/txgemma-9b-chat-GGUF/resolve/main/txgemma-9b-chat-q4_k_m.gguf) | ![Q4_K_M](https://img.shields.io/badge/Q4_K_M-673AB7) ⭐ | Best balance | +| [Download](https://huggingface.co/matrixportal/txgemma-9b-chat-GGUF/resolve/main/txgemma-9b-chat-q5_0.gguf) | ![Q5_0](https://img.shields.io/badge/Q5_0-FF6D01) | Good quality | +| [Download](https://huggingface.co/matrixportal/txgemma-9b-chat-GGUF/resolve/main/txgemma-9b-chat-q5_k_s.gguf) | ![Q5_K_S](https://img.shields.io/badge/Q5_K_S-0F9D58) | Balanced | +| [Download](https://huggingface.co/matrixportal/txgemma-9b-chat-GGUF/resolve/main/txgemma-9b-chat-q5_k_m.gguf) | ![Q5_K_M](https://img.shields.io/badge/Q5_K_M-0F9D58) | High quality | +| [Download](https://huggingface.co/matrixportal/txgemma-9b-chat-GGUF/resolve/main/txgemma-9b-chat-q6_k.gguf) | ![Q6_K](https://img.shields.io/badge/Q6_K-4285F4) 🏆 | Very good quality | +| [Download](https://huggingface.co/matrixportal/txgemma-9b-chat-GGUF/resolve/main/txgemma-9b-chat-q8_0.gguf) | ![Q8_0](https://img.shields.io/badge/Q8_0-EA4335) ⚡ | Fast, best quality | +| [Download](https://huggingface.co/matrixportal/txgemma-9b-chat-GGUF/resolve/main/txgemma-9b-chat-f16.gguf) | ![F16](https://img.shields.io/badge/F16-000000) | Maximum accuracy | + +💡 **Tip:** Use `F16` for maximum precision when quality is critical + +# GGUF Model Quantization & Usage Guide with llama.cpp + +## What is GGUF and Quantization? + +**GGUF** (GPT-Generated Unified Format) is an efficient model file format developed by the `llama.cpp` team that: +- Supports multiple quantization levels +- Works cross-platform +- Enables fast loading and inference + +**Quantization** converts model weights to lower precision data types (e.g., 4-bit integers instead of 32-bit floats) to: +- Reduce model size +- Decrease memory usage +- Speed up inference +- (With minor accuracy trade-offs) + +## Step-by-Step Guide + +### 1. Prerequisites + +```bash +# System updates +sudo apt update && sudo apt upgrade -y + +# Dependencies +sudo apt install -y build-essential cmake python3-pip + +# Clone and build llama.cpp +git clone https://github.com/ggerganov/llama.cpp +cd llama.cpp +make -j4 +``` + +### 2. Using Quantized Models from Hugging Face + +My automated quantization script produces models in this format: +``` +https://huggingface.co/matrixportal/txgemma-9b-chat-GGUF/resolve/main/txgemma-9b-chat-q4_k_m.gguf +``` + +Download your quantized model directly: + +```bash +wget https://huggingface.co/matrixportal/txgemma-9b-chat-GGUF/resolve/main/txgemma-9b-chat-q4_k_m.gguf +``` + +### 3. Running the Quantized Model + +Basic usage: +```bash +./main -m txgemma-9b-chat-q4_k_m.gguf -p "Your prompt here" -n 128 +``` + +Example with a creative writing prompt: +```bash +./main -m txgemma-9b-chat-q4_k_m.gguf -p "[INST] Write a short poem about AI quantization in the style of Shakespeare [/INST]" -n 256 -c 2048 -t 8 --temp 0.7 +``` + +Advanced parameters: +```bash +./main -m txgemma-9b-chat-q4_k_m.gguf -p "Question: What is the GGUF format? +Answer:" -n 256 -c 2048 -t 8 --temp 0.7 --top-k 40 --top-p 0.9 +``` + +### 4. Python Integration + +Install the Python package: +```bash +pip install llama-cpp-python +``` + +Example script: +```python +from llama_cpp import Llama + +# Initialize the model +llm = Llama( + model_path="txgemma-9b-chat-q4_k_m.gguf", + n_ctx=2048, + n_threads=8 +) + +# Run inference +response = llm( + "[INST] Explain GGUF quantization to a beginner [/INST]", + max_tokens=256, + temperature=0.7, + top_p=0.9 +) + +print(response["choices"][0]["text"]) +``` + +## Performance Tips + +1. **Hardware Utilization**: + - Set thread count with `-t` (typically CPU core count) + - Compile with CUDA/OpenCL for GPU support + +2. **Memory Optimization**: + - Lower quantization (like q4_k_m) uses less RAM + - Adjust context size with `-c` parameter + +3. **Speed/Accuracy Balance**: + - Higher bit quantization is slower but more accurate + - Reduce randomness with `--temp 0` for consistent results + +## FAQ + +**Q: What quantization levels are available?** +A: Common options include q4_0, q4_k_m, q5_0, q5_k_m, q8_0 + +**Q: How much performance loss occurs with q4_k_m?** +A: Typically 2-5% accuracy reduction but 4x smaller size + +**Q: How to enable GPU support?** +A: Build with `make LLAMA_CUBLAS=1` for NVIDIA GPUs + +## Useful Resources + +1. [llama.cpp GitHub](https://github.com/ggerganov/llama.cpp) +2. [GGUF Format Specs](https://github.com/ggerganov/ggml/blob/master/docs/gguf.md) +3. [Hugging Face Model Hub](https://huggingface.co/models) diff --git a/txgemma-9b-chat-f16.gguf b/txgemma-9b-chat-f16.gguf new file mode 100644 index 0000000..0402253 --- /dev/null +++ b/txgemma-9b-chat-f16.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:51383c2a146d180937df18759b370ea50b688d52b46fcb369de5222bca662208 +size 18490680512 diff --git a/txgemma-9b-chat-q2_k.gguf b/txgemma-9b-chat-q2_k.gguf new file mode 100644 index 0000000..24c409b --- /dev/null +++ b/txgemma-9b-chat-q2_k.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9a1cbcdb468f6df198a5f28eed23a0743065c2cf6b8d9eb56a09ce010fb7f710 +size 3805398208 diff --git a/txgemma-9b-chat-q3_k_l.gguf b/txgemma-9b-chat-q3_k_l.gguf new file mode 100644 index 0000000..1698dc9 --- /dev/null +++ b/txgemma-9b-chat-q3_k_l.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8d3f85a2715d288be6fe6a8ba8dee3dcddc1db64c03eaf38d8964ce3d76f292c +size 5132453056 diff --git a/txgemma-9b-chat-q3_k_m.gguf b/txgemma-9b-chat-q3_k_m.gguf new file mode 100644 index 0000000..0d3d414 --- /dev/null +++ b/txgemma-9b-chat-q3_k_m.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:30dc1be4e41a9c0172c91fae68373fe60aaa5bd0e73b767ce4061282a65a0565 +size 4761781440 diff --git a/txgemma-9b-chat-q3_k_s.gguf b/txgemma-9b-chat-q3_k_s.gguf new file mode 100644 index 0000000..efe40a8 --- /dev/null +++ b/txgemma-9b-chat-q3_k_s.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:24e4f5367d74baf79463919461e310ac5e5ac027a65b7d0ffba413c0168642a3 +size 4337665216 diff --git a/txgemma-9b-chat-q4_0.gguf b/txgemma-9b-chat-q4_0.gguf new file mode 100644 index 0000000..9b33268 --- /dev/null +++ b/txgemma-9b-chat-q4_0.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5748dc6378aa8644c39ceb88a278824d8054f6c18765693dbe26cc2743e5eeec +size 5443142848 diff --git a/txgemma-9b-chat-q4_k_m.gguf b/txgemma-9b-chat-q4_k_m.gguf new file mode 100644 index 0000000..ae17bc2 --- /dev/null +++ b/txgemma-9b-chat-q4_k_m.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bef67b1498619c01750e78eebb72cc270768c45f9b44b97aca4535ed90274c70 +size 5761057984 diff --git a/txgemma-9b-chat-q4_k_s.gguf b/txgemma-9b-chat-q4_k_s.gguf new file mode 100644 index 0000000..0c29a90 --- /dev/null +++ b/txgemma-9b-chat-q4_k_s.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4c5d4fceca2ad3abacfe370f5feadec197f22622e9f1a6e8bdf28e90f40add0d +size 5478925504 diff --git a/txgemma-9b-chat-q5_0.gguf b/txgemma-9b-chat-q5_0.gguf new file mode 100644 index 0000000..d6992b2 --- /dev/null +++ b/txgemma-9b-chat-q5_0.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:99173a0eecd436d0f0ba8ad6e2e80a80905d391025533c956e304019380a5ead +size 6483592384 diff --git a/txgemma-9b-chat-q5_k_m.gguf b/txgemma-9b-chat-q5_k_m.gguf new file mode 100644 index 0000000..32b99b0 --- /dev/null +++ b/txgemma-9b-chat-q5_k_m.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5580222f79e958b2f9f1eb1079cec5e18bd04544c3739d663e90bd9cb87305c +size 6647366848 diff --git a/txgemma-9b-chat-q5_k_s.gguf b/txgemma-9b-chat-q5_k_s.gguf new file mode 100644 index 0000000..9c1f337 --- /dev/null +++ b/txgemma-9b-chat-q5_k_s.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2d07516c99bc5c4c275d81481ecd1b6b79e4f0ca960f0cea9afb72bd322bc5d7 +size 6483592384 diff --git a/txgemma-9b-chat-q6_k.gguf b/txgemma-9b-chat-q6_k.gguf new file mode 100644 index 0000000..38f4fc2 --- /dev/null +++ b/txgemma-9b-chat-q6_k.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e8ecfa7e8461aa0a0b90877700659a9665cd511f22c30dcde83f48792ee0c21f +size 7589070016 diff --git a/txgemma-9b-chat-q8_0.gguf b/txgemma-9b-chat-q8_0.gguf new file mode 100644 index 0000000..8cbaf8d --- /dev/null +++ b/txgemma-9b-chat-q8_0.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:26b5be0b0784ccbfd2179ab8cbe0909d5113a4ce3bbeac06e75a1335b2666c16 +size 9827148992