From ed1ab713dc03381fbd4ec8c41de2cffbfb5e4959 Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Wed, 19 Aug 2026 10:35:16 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: Geraldine/Gemini-Distill-Qwen2.5-0.5B-ead-GGUF Source: Original Platform --- .gitattributes | 43 +++++++ Gemini-Distill-Qwen2.5-0.5B-ead-Q2_K.gguf | 3 + Gemini-Distill-Qwen2.5-0.5B-ead-Q3_K_M.gguf | 3 + Gemini-Distill-Qwen2.5-0.5B-ead-Q4_K_M.gguf | 3 + Gemini-Distill-Qwen2.5-0.5B-ead-Q5_K_M.gguf | 3 + Gemini-Distill-Qwen2.5-0.5B-ead-Q6_K.gguf | 3 + Gemini-Distill-Qwen2.5-0.5B-ead-Q8_0.gguf | 3 + Gemini-Distill-Qwen2.5-0.5B-ead-fp16.gguf | 3 + Gemini-Distill-Qwen2.5-0.5B-ead-fp32.gguf | 3 + README.md | 125 ++++++++++++++++++++ 10 files changed, 192 insertions(+) create mode 100644 .gitattributes create mode 100644 Gemini-Distill-Qwen2.5-0.5B-ead-Q2_K.gguf create mode 100644 Gemini-Distill-Qwen2.5-0.5B-ead-Q3_K_M.gguf create mode 100644 Gemini-Distill-Qwen2.5-0.5B-ead-Q4_K_M.gguf create mode 100644 Gemini-Distill-Qwen2.5-0.5B-ead-Q5_K_M.gguf create mode 100644 Gemini-Distill-Qwen2.5-0.5B-ead-Q6_K.gguf create mode 100644 Gemini-Distill-Qwen2.5-0.5B-ead-Q8_0.gguf create mode 100644 Gemini-Distill-Qwen2.5-0.5B-ead-fp16.gguf create mode 100644 Gemini-Distill-Qwen2.5-0.5B-ead-fp32.gguf create mode 100644 README.md diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..dd1daf9 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,43 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +Gemini-Distill-Qwen2.5-0.5B-ead-fp16.gguf filter=lfs diff=lfs merge=lfs -text +Gemini-Distill-Qwen2.5-0.5B-ead-fp32.gguf filter=lfs diff=lfs merge=lfs -text +Gemini-Distill-Qwen2.5-0.5B-ead-Q3_K_M.gguf filter=lfs diff=lfs merge=lfs -text +Gemini-Distill-Qwen2.5-0.5B-ead-Q4_K_M.gguf filter=lfs diff=lfs merge=lfs -text +Gemini-Distill-Qwen2.5-0.5B-ead-Q5_K_M.gguf filter=lfs diff=lfs merge=lfs -text +Gemini-Distill-Qwen2.5-0.5B-ead-Q2_K.gguf filter=lfs diff=lfs merge=lfs -text +Gemini-Distill-Qwen2.5-0.5B-ead-Q6_K.gguf filter=lfs diff=lfs merge=lfs -text +Gemini-Distill-Qwen2.5-0.5B-ead-Q8_0.gguf filter=lfs diff=lfs merge=lfs -text diff --git a/Gemini-Distill-Qwen2.5-0.5B-ead-Q2_K.gguf b/Gemini-Distill-Qwen2.5-0.5B-ead-Q2_K.gguf new file mode 100644 index 0000000..4d45f7f --- /dev/null +++ b/Gemini-Distill-Qwen2.5-0.5B-ead-Q2_K.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fdeb4f1ea8a1249a4ffa2e7c4bbe55e1fb8a6979eb1e50d0a369511e6afc1c67 +size 338346656 diff --git a/Gemini-Distill-Qwen2.5-0.5B-ead-Q3_K_M.gguf b/Gemini-Distill-Qwen2.5-0.5B-ead-Q3_K_M.gguf new file mode 100644 index 0000000..b3d942d --- /dev/null +++ b/Gemini-Distill-Qwen2.5-0.5B-ead-Q3_K_M.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:782cb5dc63d9fb2c3aaf8693a609ca9ce5af48e228981025ee9137b4f55bee49 +size 355205792 diff --git a/Gemini-Distill-Qwen2.5-0.5B-ead-Q4_K_M.gguf b/Gemini-Distill-Qwen2.5-0.5B-ead-Q4_K_M.gguf new file mode 100644 index 0000000..44308b5 --- /dev/null +++ b/Gemini-Distill-Qwen2.5-0.5B-ead-Q4_K_M.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be851428a84baa6ade53e09a052cff53b99f596ecb7a5434671cc6014fa72ca9 +size 397547168 diff --git a/Gemini-Distill-Qwen2.5-0.5B-ead-Q5_K_M.gguf b/Gemini-Distill-Qwen2.5-0.5B-ead-Q5_K_M.gguf new file mode 100644 index 0000000..b26fe09 --- /dev/null +++ b/Gemini-Distill-Qwen2.5-0.5B-ead-Q5_K_M.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8166fbe2ff0f94e9d3238ddf39040c3829667c33977b1e53fbba8f0b4081c40b +size 419825312 diff --git a/Gemini-Distill-Qwen2.5-0.5B-ead-Q6_K.gguf b/Gemini-Distill-Qwen2.5-0.5B-ead-Q6_K.gguf new file mode 100644 index 0000000..959b227 --- /dev/null +++ b/Gemini-Distill-Qwen2.5-0.5B-ead-Q6_K.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1455b24a8436a11714dc4546d337372d72547ccedf8f9c83f6e67259e4696a8a +size 505475744 diff --git a/Gemini-Distill-Qwen2.5-0.5B-ead-Q8_0.gguf b/Gemini-Distill-Qwen2.5-0.5B-ead-Q8_0.gguf new file mode 100644 index 0000000..1b9224f --- /dev/null +++ b/Gemini-Distill-Qwen2.5-0.5B-ead-Q8_0.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9720305520484387cf6eb91314b981e9a9a35c643a518164b3ae9cdb923081e7 +size 530807456 diff --git a/Gemini-Distill-Qwen2.5-0.5B-ead-fp16.gguf b/Gemini-Distill-Qwen2.5-0.5B-ead-fp16.gguf new file mode 100644 index 0000000..b47eeab --- /dev/null +++ b/Gemini-Distill-Qwen2.5-0.5B-ead-fp16.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:49a48da792516b224a4e5eb7eda2835bd8fb3c46fd4c5eb64f077d59f27fb373 +size 993671808 diff --git a/Gemini-Distill-Qwen2.5-0.5B-ead-fp32.gguf b/Gemini-Distill-Qwen2.5-0.5B-ead-fp32.gguf new file mode 100644 index 0000000..4713fa0 --- /dev/null +++ b/Gemini-Distill-Qwen2.5-0.5B-ead-fp32.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2683df3097d6482f84014e3c3281c5567248288de200caf4a59c37476fdab190 +size 1981115776 diff --git a/README.md b/README.md new file mode 100644 index 0000000..7b4d5db --- /dev/null +++ b/README.md @@ -0,0 +1,125 @@ +--- +license: mit +language: +- fr +- en +base_model: +- Geraldine/Gemini-Distill-Qwen2.5-0.5B-ead +--- + +# Gemini-Distill-Qwen2.5-0.5B-ead GGUF Quantized Versions (Distilled from Gemini-2.0-Flash-Thinking-Exp) + +## Model Description +This repository contains **quantized versions** of the fine-tuned **Geraldine/Gemini-Distill-Qwen2.5-0.5B-ead** model, which was trained via knowledge distillation from **Gemini-2.0-Flash-Thinking-Exp**. The fine-tuning process teaches the model to reason through and generate **Encoded Archival Description (EAD/XML)** outputs, ensuring structured reasoning before final archival XML generation. + +This repository provides various **GGUF quantized formats**, allowing efficient inference on different hardware setups, including CPUs and GPUs. + +--- +## Available GGUF Files + +The following quantized versions of the model were generated using **llama.cpp**: + +| File Name | Description | +|-----------|-------------| +| `Gemini-Distill-Qwen2.5-0.5B-ead-Q2_K.gguf` | Ultra-low precision (2-bit) for extreme compression | +| `Gemini-Distill-Qwen2.5-0.5B-ead-Q3_K_M.gguf` | 3-bit quantization with mixed precision | +| `Gemini-Distill-Qwen2.5-0.5B-ead-Q4_K_M.gguf` | 4-bit quantization with mixed precision | +| `Gemini-Distill-Qwen2.5-0.5B-ead-Q5_K_M.gguf` | 5-bit quantization with mixed precision | +| `Gemini-Distill-Qwen2.5-0.5B-ead-Q6_K.gguf` | 6-bit quantization | +| `Gemini-Distill-Qwen2.5-0.5B-ead-Q8_0.gguf` | 8-bit quantization for balance between speed and accuracy | +| `Gemini-Distill-Qwen2.5-0.5B-ead-fp16.gguf` | 16-bit floating point (fp16) version | +| `Gemini-Distill-Qwen2.5-0.5B-ead-fp32.gguf` | Full precision (fp32) version | + +--- +## How to Use the Quantized Model + +### **Running the Model with llama.cpp** +To run the model using `llama.cpp`, use the following command: + +```bash +./main -m Gemini-Distill-Qwen2.5-0.5B-ead-Q4_K_M.gguf -p "Convert the following archival information into EAD/XML: ..." +``` + +For optimal performance, ensure you select the right quantized version based on your hardware capabilities. + +### **Running the Model with GPT4All** +If using GPT4All, load the GGUF model with: + +```python +from gpt4all import GPT4All + +model_path = "Gemini-Distill-Qwen2.5-0.5B-ead-Q4_K_M.gguf" +model = GPT4All(model_path) +response = model.generate("Convert the following archival information into EAD/XML:") +print(response) +``` + +### **Running the Model with Ollama** +If using Ollama, load the GGUF model with: + +```bash +ollama run hf.co/Geraldine/Gemini-Distill-Qwen2.5-0.5B-ead-GGUF:Q8_0 +``` + +```python +import requests +import json + +url = "http://localhost:11434/v1/chat/completions" + +payload = json.dumps({ + "model": "hf.co/Geraldine/Gemini-Distill-Qwen2.5-0.5B-ead-GGUF:Q8_0", + "messages": [ + { + "role": "system", + "content": "You are an archivist expert in EAD/XML format for archival records metadata." + }, + { + "role": "user", + "content": "Give me an example of content." + } + ], + "option": { + "num_ctx": 4096, + "temperature": 0.1 + }, + "stream": False +}) +headers = { + 'Content-Type': 'application/json' +} + +response = requests.request("POST", url, headers=headers, data=payload) + +print(response.text) +``` + +--- +## Choosing the Right Quantization Format +- **Lower-bit models (Q2_K, Q3_K_M, Q4_K_M):** Best for low-memory devices, but may lose some accuracy. +- **Mid-range (Q5_K_M, Q6_K):** Good trade-off between speed and precision. +- **Higher precision (Q8_0, fp16, fp32):** Best for accuracy but requires more memory. + +For CPU inference, **Q4_K_M or Q5_K_M** is recommended for a balance between efficiency and performance. + +--- +## Limitations & Future Improvements +- **Inference Speed:** Ensure **Sliding Window Attention (SWA) is disabled**, as it may slow down inference. + - To disable: `model.config.sliding_window = None` +- **Future Work:** + - Further optimizations for CPU inference + - Additional fine-tuning on larger datasets + - Exploring LoRA/QLoRA for low-rank adaptation + +--- +## Citation & Acknowledgments +If you use this model in research or production, please cite: +``` +@misc{your-citation, + author = {Géraldine Geoffroy}, + title = {Gemini-Distill-Qwen2.5-0.5B-ead GGUF Quantized Versions}, + year = {2025}, + publisher = {Hugging Face}, + url = {https://huggingface.co/Geraldine/Gemini-Distill-Qwen2.5-0.5B-ead-GGUF} +} +``` \ No newline at end of file