From ba3ae6d4110379c836506b3467c6a0c538fdf57c Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Sat, 30 May 2026 09:10:15 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: Mungert/granite-3.2-2b-instruct-GGUF Source: Original Platform --- .gitattributes | 78 +++++ README.md | 434 +++++++++++++++++++++++++ granite-3.2-2b-instruct-bf16-q4_k.gguf | 3 + granite-3.2-2b-instruct-bf16-q6_k.gguf | 3 + granite-3.2-2b-instruct-bf16-q8_0.gguf | 3 + granite-3.2-2b-instruct-bf16.gguf | 3 + granite-3.2-2b-instruct-bf16_q8_0.gguf | 3 + granite-3.2-2b-instruct-f16-q4_k.gguf | 3 + granite-3.2-2b-instruct-f16-q6_k.gguf | 3 + granite-3.2-2b-instruct-f16-q8_0.gguf | 3 + granite-3.2-2b-instruct-f16_q8_0.gguf | 3 + granite-3.2-2b-instruct-iq3_m.gguf | 3 + granite-3.2-2b-instruct-iq3_s.gguf | 3 + granite-3.2-2b-instruct-iq3_xs.gguf | 3 + granite-3.2-2b-instruct-iq3_xxs.gguf | 3 + granite-3.2-2b-instruct-iq4_nl.gguf | 3 + granite-3.2-2b-instruct-iq4_xs.gguf | 3 + granite-3.2-2b-instruct-q3_k_m.gguf | 3 + granite-3.2-2b-instruct-q3_k_s.gguf | 3 + granite-3.2-2b-instruct-q4_0.gguf | 3 + granite-3.2-2b-instruct-q4_1.gguf | 3 + granite-3.2-2b-instruct-q4_k_m.gguf | 3 + granite-3.2-2b-instruct-q4_k_s.gguf | 3 + granite-3.2-2b-instruct-q5_0.gguf | 3 + granite-3.2-2b-instruct-q5_1.gguf | 3 + granite-3.2-2b-instruct-q5_k_m.gguf | 3 + granite-3.2-2b-instruct-q5_k_s.gguf | 3 + granite-3.2-2b-instruct-q6_k_m.gguf | 3 + granite-3.2-2b-instruct-q8_0.gguf | 3 + granite-3.2-2b-instruct-tq1_0.gguf | 3 + granite-3.2-2b-instruct-tq2_0.gguf | 3 + granite-3.2-2b-instruct.imatrix | 3 + 32 files changed, 602 insertions(+) create mode 100644 .gitattributes create mode 100644 README.md create mode 100644 granite-3.2-2b-instruct-bf16-q4_k.gguf create mode 100644 granite-3.2-2b-instruct-bf16-q6_k.gguf create mode 100644 granite-3.2-2b-instruct-bf16-q8_0.gguf create mode 100644 granite-3.2-2b-instruct-bf16.gguf create mode 100644 granite-3.2-2b-instruct-bf16_q8_0.gguf create mode 100644 granite-3.2-2b-instruct-f16-q4_k.gguf create mode 100644 granite-3.2-2b-instruct-f16-q6_k.gguf create mode 100644 granite-3.2-2b-instruct-f16-q8_0.gguf create mode 100644 granite-3.2-2b-instruct-f16_q8_0.gguf create mode 100644 granite-3.2-2b-instruct-iq3_m.gguf create mode 100644 granite-3.2-2b-instruct-iq3_s.gguf create mode 100644 granite-3.2-2b-instruct-iq3_xs.gguf create mode 100644 granite-3.2-2b-instruct-iq3_xxs.gguf create mode 100644 granite-3.2-2b-instruct-iq4_nl.gguf create mode 100644 granite-3.2-2b-instruct-iq4_xs.gguf create mode 100644 granite-3.2-2b-instruct-q3_k_m.gguf create mode 100644 granite-3.2-2b-instruct-q3_k_s.gguf create mode 100644 granite-3.2-2b-instruct-q4_0.gguf create mode 100644 granite-3.2-2b-instruct-q4_1.gguf create mode 100644 granite-3.2-2b-instruct-q4_k_m.gguf create mode 100644 granite-3.2-2b-instruct-q4_k_s.gguf create mode 100644 granite-3.2-2b-instruct-q5_0.gguf create mode 100644 granite-3.2-2b-instruct-q5_1.gguf create mode 100644 granite-3.2-2b-instruct-q5_k_m.gguf create mode 100644 granite-3.2-2b-instruct-q5_k_s.gguf create mode 100644 granite-3.2-2b-instruct-q6_k_m.gguf create mode 100644 granite-3.2-2b-instruct-q8_0.gguf create mode 100644 granite-3.2-2b-instruct-tq1_0.gguf create mode 100644 granite-3.2-2b-instruct-tq2_0.gguf create mode 100644 granite-3.2-2b-instruct.imatrix diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..7d9df57 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,78 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +granite-3.2-2b-instruct-tq1_0.gguf filter=lfs diff=lfs merge=lfs -text +granite-3.2-2b-instruct-tq2_0.gguf filter=lfs diff=lfs merge=lfs -text +granite-3.2-2b-instruct-q4_1.gguf filter=lfs diff=lfs merge=lfs -text +granite-3.2-2b-instruct-q5_k_m.gguf filter=lfs diff=lfs merge=lfs -text +granite-3.2-2b-instruct-q3_k_l.gguf filter=lfs diff=lfs merge=lfs -text +granite-3.2-2b-instruct-q5_k_l.gguf filter=lfs diff=lfs merge=lfs -text +granite-3.2-2b-instruct-f16-q4_k.gguf filter=lfs diff=lfs merge=lfs -text +granite-3.2-2b-instruct-bf16-q6_k.gguf filter=lfs diff=lfs merge=lfs -text +granite-3.2-2b-instruct-q4_1_l.gguf filter=lfs diff=lfs merge=lfs -text +granite-3.2-2b-instruct-q3_k_s.gguf filter=lfs diff=lfs merge=lfs -text +granite-3.2-2b-instruct-iq3_xxs.gguf filter=lfs diff=lfs merge=lfs -text +granite-3.2-2b-instruct-bf16-q8_0.gguf filter=lfs diff=lfs merge=lfs -text +granite-3.2-2b-instruct-q4_0.gguf filter=lfs diff=lfs merge=lfs -text +granite-3.2-2b-instruct-iq3_xs.gguf filter=lfs diff=lfs merge=lfs -text +granite-3.2-2b-instruct-q6_k_m.gguf filter=lfs diff=lfs merge=lfs -text +granite-3.2-2b-instruct-q5_1.gguf filter=lfs diff=lfs merge=lfs -text +granite-3.2-2b-instruct.imatrix filter=lfs diff=lfs merge=lfs -text +granite-3.2-2b-instruct-q5_k_s.gguf filter=lfs diff=lfs merge=lfs -text +granite-3.2-2b-instruct-f16-q8_0.gguf filter=lfs diff=lfs merge=lfs -text +granite-3.2-2b-instruct-iq3_m.gguf filter=lfs diff=lfs merge=lfs -text +granite-3.2-2b-instruct-iq4_nl.gguf filter=lfs diff=lfs merge=lfs -text +granite-3.2-2b-instruct-q5_1_l.gguf filter=lfs diff=lfs merge=lfs -text +granite-3.2-2b-instruct-q3_k_m.gguf filter=lfs diff=lfs merge=lfs -text +granite-3.2-2b-instruct-q4_k_s.gguf filter=lfs diff=lfs merge=lfs -text +granite-3.2-2b-instruct-q6_k_l.gguf filter=lfs diff=lfs merge=lfs -text +granite-3.2-2b-instruct-q4_0_l.gguf filter=lfs diff=lfs merge=lfs -text +granite-3.2-2b-instruct-q5_0.gguf filter=lfs diff=lfs merge=lfs -text +granite-3.2-2b-instruct-bf16.gguf filter=lfs diff=lfs merge=lfs -text +granite-3.2-2b-instruct-f16.gguf filter=lfs diff=lfs merge=lfs -text +granite-3.2-2b-instruct-iq4_xs.gguf filter=lfs diff=lfs merge=lfs -text +granite-3.2-2b-instruct-q4_k_m.gguf filter=lfs diff=lfs merge=lfs -text +granite-3.2-2b-instruct-q4_k_l.gguf filter=lfs diff=lfs merge=lfs -text +granite-3.2-2b-instruct-q5_0_l.gguf filter=lfs diff=lfs merge=lfs -text +granite-3.2-2b-instruct-q8_0.gguf filter=lfs diff=lfs merge=lfs -text +granite-3.2-2b-instruct-iq3_s.gguf filter=lfs diff=lfs merge=lfs -text +granite-3.2-2b-instruct-f16-q6_k.gguf filter=lfs diff=lfs merge=lfs -text +granite-3.2-2b-instruct-bf16-q4_k.gguf filter=lfs diff=lfs merge=lfs -text +granite-3.2-2b-instruct-f16_q8_0.gguf filter=lfs diff=lfs merge=lfs -text +granite-3.2-2b-instruct-bf16_q8_0.gguf filter=lfs diff=lfs merge=lfs -text +granite-3.2-2b-instruct-f16_q6_k.gguf filter=lfs diff=lfs merge=lfs -text +granite-3.2-2b-instruct-bf16_q6_k.gguf filter=lfs diff=lfs merge=lfs -text +granite-3.2-2b-instruct-f16_q4_k.gguf filter=lfs diff=lfs merge=lfs -text +granite-3.2-2b-instruct-bf16_q4_k.gguf filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..c01f1bd --- /dev/null +++ b/README.md @@ -0,0 +1,434 @@ +--- +pipeline_tag: text-generation +inference: false +license: apache-2.0 +library_name: transformers +tags: +- language +- granite-3.2 +base_model: +- ibm-granite/granite-3.1-2b-instruct +new_version: ibm-granite/granite-3.3-2b-instruct +--- + +# granite-3.2-2b-instruct GGUF Models + + +## Model Generation Details + +This model was generated using [llama.cpp](https://github.com/ggerganov/llama.cpp) at commit [`5dd942de`](https://github.com/ggerganov/llama.cpp/commit/5dd942de5922a22ec8446a4ad2203738dbcb9389). + + + + + + +--- + + + Click here to get info on choosing the right GGUF model format + + +--- + + + + + + +# Granite-3.2-2B-Instruct + +**Model Summary:** +Granite-3.2-2B-Instruct is an 2-billion-parameter, long-context AI model fine-tuned for thinking capabilities. Built on top of [Granite-3.1-2B-Instruct](https://huggingface.co/ibm-granite/granite-3.1-2b-instruct), it has been trained using a mix of permissively licensed open-source datasets and internally generated synthetic data designed for reasoning tasks. The model allows controllability of its thinking capability, ensuring it is applied only when required. + + +- **Developers:** Granite Team, IBM +- **Website**: [Granite Docs](https://www.ibm.com/granite/docs/) +- **Release Date**: February 26th, 2025 +- **License:** [Apache 2.0](https://www.apache.org/licenses/LICENSE-2.0) + +**Supported Languages:** +English, German, Spanish, French, Japanese, Portuguese, Arabic, Czech, Italian, Korean, Dutch, and Chinese. However, users may finetune this Granite model for languages beyond these 12 languages. + +**Intended Use:** +This model is designed to handle general instruction-following tasks and can be integrated into AI assistants across various domains, including business applications. + +**Capabilities** +* **Thinking** +* Summarization +* Text classification +* Text extraction +* Question-answering +* Retrieval Augmented Generation (RAG) +* Code related tasks +* Function-calling tasks +* Multilingual dialog use cases +* Long-context tasks including long document/meeting summarization, long document QA, etc. + + + +**Generation:** +This is a simple example of how to use Granite-3.2-2B-Instruct model. + +Install the following libraries: + +```shell +pip install torch torchvision torchaudio +pip install accelerate +pip install transformers +``` +Then, copy the snippet from the section that is relevant for your use case. + +```python +from transformers import AutoModelForCausalLM, AutoTokenizer, set_seed +import torch + +model_path="ibm-granite/granite-3.2-2b-instruct" +device="cuda" +model = AutoModelForCausalLM.from_pretrained( + model_path, + device_map=device, + torch_dtype=torch.bfloat16, + ) +tokenizer = AutoTokenizer.from_pretrained( + model_path +) + +conv = [{"role": "user", "content":"You have 10 liters of a 30% acid solution. How many liters of a 70% acid solution must be added to achieve a 50% acid mixture?"}] + +input_ids = tokenizer.apply_chat_template(conv, return_tensors="pt", thinking=True, return_dict=True, add_generation_prompt=True).to(device) + +set_seed(42) +output = model.generate( + **input_ids, + max_new_tokens=8192, +) + +prediction = tokenizer.decode(output[0, input_ids["input_ids"].shape[1]:], skip_special_tokens=True) +print(prediction) +``` + +**Example Outputs** +- thinking=True +```md +Here is my thought process: + +1. First, let's understand the composition of the initial 10 liters of 30% acid solution: + - Acid content = 30% of 10 liters = 0.30 * 10 = 3 liters + +2. We want to end up with a 50% acid solution. Let's denote the volume of the 70% acid solution to be added as 'x' liters. + +3. The amount of acid in the final mixture (50% solution) should equal the initial acid content plus the acid from the added 70% solution. + - Total acid in final mixture = 0.50 * (10 + x) + +4. The equation representing the total acid content is: + 3 (from initial solution) + 0.70 * x (from added solution) = 0.50 * (10 + x) + +5. Now, we solve for 'x': + 3 + 0.70x = 5 + 0.50x + 0.70x - 0.50x = 5 - 3 + 0.20x = 2 + x = 2 / 0.20 + x = 10 liters + +6. Therefore, 10 liters of a 70% acid solution must be added to the initial 10 liters of 30% acid solution to achieve a 50% acid mixture. + +Here is my response: + +To solve this problem, we need to determine how much of a 70% acid solution must be added to a 10-liter solution that is already 30% acid to create a new mixture that is 50% acid. Here's how to calculate it: + +1. **Initial Acid Content**: + - The initial 10-liter solution contains 3 liters of acid (since 30% of 10 liters). + +2. **Desired Final Acid Content**: + - We want the final mixture to be 50% acid. + +3. **Equation Setup**: + - Let 'x' be the volume (in liters) of the 70% acid solution to be added. + - The equation representing the total acid content in the final mixture (which is 10 liters from the initial solution plus 'x' liters of the added solution) should equal 50% of the total volume (10 + x). + - This gives us: 3 (acid from initial solution) + 0.70x (acid from added solution) = 0.50(10 + x) + +4. **Solving the Equation**: + - First, simplify the right side of the equation: 0.50(10 + x) = 5 + 0.50x + - Now, the equation is: 3 + 0.70x = 5 + 0.50x + - Subtract 0.50x from both sides: 0.20x = 2 + - Divide both sides by 0.20: x = 2 / 0.20 + - x = 10 liters + +5. **Conclusion**: + - To achieve a 50% acid mixture from the initial 30% solution, you need to add 10 liters of a 70% acid solution. + +So, the final answer is that 10 liters of a 70% acid solution must be added to the initial 10 liters of 30% acid solution. +``` +- thinking=False +```md +To solve this problem, we'll use the concept of volume and concentration. Let's denote the amount of 70% acid solution we need to add as "x" liters. + +First, let's find out how much acid is in the initial 10-liter solution: + +Acid in initial solution = 30% of 10 liters = 0.30 * 10 = 3 liters + +Now, we want to end up with a 50% acid solution in a total volume of (10 + x) liters. Let's denote the final volume as V. + +Final acid concentration = 50% +Final acid amount = 50% of V = 0.50 * V + +We know the initial acid amount and the final acid amount, so we can set up an equation: + +Initial acid amount + Acid from added solution = Final acid amount +3 liters + (70% of x) = 0.50 * (10 + x) + +Now, let's solve for x: + +0.70x + 3 = 0.50 * 10 + 0.50x +0.70x - 0.50x = 0.50 * 10 - 3 +0.20x = 5 - 3 +0.20x = 2 +x = 2 / 0.20 +x = 10 liters + +So, you need to add 10 liters of a 70% acid solution to the initial 10-liter 30% acid solution to achieve a 50% acid mixture. +``` + +**Evaluation Results:** + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ModelsArenaHardAlpaca-Eval-2MMLUPopQATruthfulQABigBenchHardDROPGSM8KHumanEvalHumanEval+IFEvalAttaQ
Llama-3.1-8B-Instruct36.4327.2269.1528.7952.7972.6661.4883.2485.3280.1579.1083.43
DeepSeek-R1-Distill-Llama-8B17.1721.8545.8013.2547.4365.7144.4672.1867.5462.9166.5042.87
Qwen-2.5-7B-Instruct25.4430.3474.3018.1263.0670.4054.7184.4693.3589.9174.9081.90
DeepSeek-R1-Distill-Qwen-7B10.3615.3550.729.9447.1465.0442.7678.4779.8978.4359.1042.45
Granite-3.1-8B-Instruct37.5830.3466.7728.765.8468.5550.7879.1589.6385.7973.2085.73
Granite-3.1-2B-Instruct23.327.1757.1120.5559.7954.4618.6867.5579.4575.2663.5984.7
Granite-3.2-8B-Instruct55.2561.1966.7928.0466.9264.7750.9581.6589.3585.7274.3185.42
Granite-3.2-2B-Instruct24.8634.5157.1820.5659.852.2721.1267.0280.1373.3961.5583.23
+ +**Training Data:** +Overall, our training data is largely comprised of two key sources: (1) publicly available datasets with permissive license, (2) internal synthetically generated data targeted to enhance reasoning capabilites. + + +**Infrastructure:** +We train Granite-3.2-2B-Instruct using IBM's super computing cluster, Blue Vela, which is outfitted with NVIDIA H100 GPUs. This cluster provides a scalable and efficient infrastructure for training our models over thousands of GPUs. + +**Ethical Considerations and Limitations:** +Granite-3.2-2B-Instruct builds upon Granite-3.1-2B-Instruct, leveraging both permissively licensed open-source and select proprietary data for enhanced performance. Since it inherits its foundation from the previous model, all ethical considerations and limitations applicable to [Granite-3.1-2B-Instruct](https://huggingface.co/ibm-granite/granite-3.1-2b-instruct) remain relevant. + + +**Resources** +- ⭐️ Learn about the latest updates with Granite: https://www.ibm.com/granite +- πŸ“„ Get started with tutorials, best practices, and prompt engineering advice: https://www.ibm.com/granite/docs/ +- πŸ’‘ Learn about the latest Granite learning resources: https://ibm.biz/granite-learning-resources + + + + + +--- + +# πŸš€ If you find these models useful + +Help me test my **AI-Powered Quantum Network Monitor Assistant** with **quantum-ready security checks**: + +πŸ‘‰ [Quantum Network Monitor](https://readyforquantum.com/?assistant=open&utm_source=huggingface&utm_medium=referral&utm_campaign=huggingface_repo_readme) + + +The full Open Source Code for the Quantum Network Monitor Service available at my github repos ( repos with NetworkMonitor in the name) : [Source Code Quantum Network Monitor](https://github.com/Mungert69). You will also find the code I use to quantize the models if you want to do it yourself [GGUFModelBuilder](https://github.com/Mungert69/GGUFModelBuilder) + +πŸ’¬ **How to test**: + Choose an **AI assistant type**: + - `TurboLLM` (GPT-4.1-mini) + - `HugLLM` (Hugginface Open-source models) + - `TestLLM` (Experimental CPU-only) + +### **What I’m Testing** +I’m pushing the limits of **small open-source models for AI network monitoring**, specifically: +- **Function calling** against live network services +- **How small can a model go** while still handling: + - Automated **Nmap security scans** + - **Quantum-readiness checks** + - **Network Monitoring tasks** + +🟑 **TestLLM** – Current experimental model (llama.cpp on 2 CPU threads on huggingface docker space): +- βœ… **Zero-configuration setup** +- ⏳ 30s load time (slow inference but **no API costs**) . No token limited as the cost is low. +- πŸ”§ **Help wanted!** If you’re into **edge-device AI**, let’s collaborate! + +### **Other Assistants** +🟒 **TurboLLM** – Uses **gpt-4.1-mini** : +- **It performs very well but unfortunatly OpenAI charges per token. For this reason tokens usage is limited. +- **Create custom cmd processors to run .net code on Quantum Network Monitor Agents** +- **Real-time network diagnostics and monitoring** +- **Security Audits** +- **Penetration testing** (Nmap/Metasploit) + +πŸ”΅ **HugLLM** – Latest Open-source models: +- 🌐 Runs on Hugging Face Inference API. Performs pretty well using the lastest models hosted on Novita. + +### πŸ’‘ **Example commands you could test**: +1. `"Give me info on my websites SSL certificate"` +2. `"Check if my server is using quantum safe encyption for communication"` +3. `"Run a comprehensive security audit on my server"` +4. '"Create a cmd processor to .. (what ever you want)" Note you need to install a [Quantum Network Monitor Agent](https://readyforquantum.com/Download/?utm_source=huggingface&utm_medium=referral&utm_campaign=huggingface_repo_readme) to run the .net code on. This is a very flexible and powerful feature. Use with caution! + +### Final Word + +I fund the servers used to create these model files, run the Quantum Network Monitor service, and pay for inference from Novita and OpenAIβ€”all out of my own pocket. All the code behind the model creation and the Quantum Network Monitor project is [open source](https://github.com/Mungert69). Feel free to use whatever you find helpful. + +If you appreciate the work, please consider [buying me a coffee](https://www.buymeacoffee.com/mahadeva) β˜•. Your support helps cover service costs and allows me to raise token limits for everyone. + +I'm also open to job opportunities or sponsorship. + +Thank you! 😊 diff --git a/granite-3.2-2b-instruct-bf16-q4_k.gguf b/granite-3.2-2b-instruct-bf16-q4_k.gguf new file mode 100644 index 0000000..8f007df --- /dev/null +++ b/granite-3.2-2b-instruct-bf16-q4_k.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1ab4beaa55f74c4179399b7f21d48ac99be5ae1fa2bcd3cb6d89e698a33b5bdc +size 1664054944 diff --git a/granite-3.2-2b-instruct-bf16-q6_k.gguf b/granite-3.2-2b-instruct-bf16-q6_k.gguf new file mode 100644 index 0000000..0ed23c0 --- /dev/null +++ b/granite-3.2-2b-instruct-bf16-q6_k.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:08215ddbcb4d81231e7810929dab71c80123a70cf6999c47e9304114d1a17c79 +size 2199320224 diff --git a/granite-3.2-2b-instruct-bf16-q8_0.gguf b/granite-3.2-2b-instruct-bf16-q8_0.gguf new file mode 100644 index 0000000..28b84ef --- /dev/null +++ b/granite-3.2-2b-instruct-bf16-q8_0.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1bfea47c7d85569c14c9af5f3cdea0be29e3c9c17d1326f1218a9e9d55d75c0e +size 2788488544 diff --git a/granite-3.2-2b-instruct-bf16.gguf b/granite-3.2-2b-instruct-bf16.gguf new file mode 100644 index 0000000..3ab5f53 --- /dev/null +++ b/granite-3.2-2b-instruct-bf16.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ddefb40c4c986d789d18d69c9b93cc77df67e7b657a70bc6b909194de3c20269 +size 5069141600 diff --git a/granite-3.2-2b-instruct-bf16_q8_0.gguf b/granite-3.2-2b-instruct-bf16_q8_0.gguf new file mode 100644 index 0000000..5fc530f --- /dev/null +++ b/granite-3.2-2b-instruct-bf16_q8_0.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b0d2328f1893dce9b6348095d5e7039cd025ec8b4ce4fbec931b2f1b75464640 +size 3614242400 diff --git a/granite-3.2-2b-instruct-f16-q4_k.gguf b/granite-3.2-2b-instruct-f16-q4_k.gguf new file mode 100644 index 0000000..6cc06cc --- /dev/null +++ b/granite-3.2-2b-instruct-f16-q4_k.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b351744138608405a6f31e79f19e9d91c1ef48a7f633c18f14a28832fe9bc09c +size 1664054944 diff --git a/granite-3.2-2b-instruct-f16-q6_k.gguf b/granite-3.2-2b-instruct-f16-q6_k.gguf new file mode 100644 index 0000000..edcfe1b --- /dev/null +++ b/granite-3.2-2b-instruct-f16-q6_k.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8e55c0cae656f360ee308d96e5900ad0013cffa24606ef7efb6096a5202d91f9 +size 2199320224 diff --git a/granite-3.2-2b-instruct-f16-q8_0.gguf b/granite-3.2-2b-instruct-f16-q8_0.gguf new file mode 100644 index 0000000..f1b4681 --- /dev/null +++ b/granite-3.2-2b-instruct-f16-q8_0.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7260e9a2d2cf73bd125d467a6c7569a23c303001d5f87d1d823353362bb81cef +size 2788488544 diff --git a/granite-3.2-2b-instruct-f16_q8_0.gguf b/granite-3.2-2b-instruct-f16_q8_0.gguf new file mode 100644 index 0000000..3501b10 --- /dev/null +++ b/granite-3.2-2b-instruct-f16_q8_0.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1f2c6ddc0d6eceb9985180f8ed5b625dc5adcc28c44f523127aae8eed272bb14 +size 3614242400 diff --git a/granite-3.2-2b-instruct-iq3_m.gguf b/granite-3.2-2b-instruct-iq3_m.gguf new file mode 100644 index 0000000..fe84ce9 --- /dev/null +++ b/granite-3.2-2b-instruct-iq3_m.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2d421e603e2386217670b4fc106b90ca729c0adf831196e1cd270bc398e2ed0d +size 1197201440 diff --git a/granite-3.2-2b-instruct-iq3_s.gguf b/granite-3.2-2b-instruct-iq3_s.gguf new file mode 100644 index 0000000..dc239cb --- /dev/null +++ b/granite-3.2-2b-instruct-iq3_s.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7f4bb48cfbbbd7d11bc25e48added78df63d4aa851dc5fbe270da392e6195535 +size 1197201440 diff --git a/granite-3.2-2b-instruct-iq3_xs.gguf b/granite-3.2-2b-instruct-iq3_xs.gguf new file mode 100644 index 0000000..d94d531 --- /dev/null +++ b/granite-3.2-2b-instruct-iq3_xs.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7c485818d9cf92a2263a17fb9d3d37a9bc8a297e117adbb969b8fd16e1fe2970 +size 1074386976 diff --git a/granite-3.2-2b-instruct-iq3_xxs.gguf b/granite-3.2-2b-instruct-iq3_xxs.gguf new file mode 100644 index 0000000..408ada4 --- /dev/null +++ b/granite-3.2-2b-instruct-iq3_xxs.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c31ae1141e8b608aec11d2f00f81a69a7de7f912f0403671ff09c1b841ca7387 +size 1045551136 diff --git a/granite-3.2-2b-instruct-iq4_nl.gguf b/granite-3.2-2b-instruct-iq4_nl.gguf new file mode 100644 index 0000000..e0f90dd --- /dev/null +++ b/granite-3.2-2b-instruct-iq4_nl.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5bf858e2a8ee38b8e7f7b3c6312dc61b38b50221bb8652d3aa0c284ef847ec9c +size 1458625376 diff --git a/granite-3.2-2b-instruct-iq4_xs.gguf b/granite-3.2-2b-instruct-iq4_xs.gguf new file mode 100644 index 0000000..bbfb057 --- /dev/null +++ b/granite-3.2-2b-instruct-iq4_xs.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e55bc5c770ba2d20ee6cc2148489da3aa9f452e3122e498216878784a70e0270 +size 1383914336 diff --git a/granite-3.2-2b-instruct-q3_k_m.gguf b/granite-3.2-2b-instruct-q3_k_m.gguf new file mode 100644 index 0000000..099fbd8 --- /dev/null +++ b/granite-3.2-2b-instruct-q3_k_m.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6029b29de8905f17ea3c00c3f32036e14b0ba43f1ceb450a1ca725812f479f5c +size 1243077472 diff --git a/granite-3.2-2b-instruct-q3_k_s.gguf b/granite-3.2-2b-instruct-q3_k_s.gguf new file mode 100644 index 0000000..fb111c0 --- /dev/null +++ b/granite-3.2-2b-instruct-q3_k_s.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1e65636db0c6061bfdfcdb2fb91208eb302e1fe155f74c9f79bd2c1fd64d6c9b +size 1229707296 diff --git a/granite-3.2-2b-instruct-q4_0.gguf b/granite-3.2-2b-instruct-q4_0.gguf new file mode 100644 index 0000000..b697e84 --- /dev/null +++ b/granite-3.2-2b-instruct-q4_0.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d41b5994af64a34ec7b2c88b88ab058eef1cb6dd38c0dce56088b92fd8b96405 +size 1427428640 diff --git a/granite-3.2-2b-instruct-q4_1.gguf b/granite-3.2-2b-instruct-q4_1.gguf new file mode 100644 index 0000000..9e48fa2 --- /dev/null +++ b/granite-3.2-2b-instruct-q4_1.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d52ee5d39e1fa450ad758edf9655ea5f886602bc8ee88505290b13e391e36612 +size 1585764000 diff --git a/granite-3.2-2b-instruct-q4_k_m.gguf b/granite-3.2-2b-instruct-q4_k_m.gguf new file mode 100644 index 0000000..78285cf --- /dev/null +++ b/granite-3.2-2b-instruct-q4_k_m.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2db5bc4ba770f23bafe3376e49acbfac82fa04b75299bff41471399712fda0aa +size 1530551136 diff --git a/granite-3.2-2b-instruct-q4_k_s.gguf b/granite-3.2-2b-instruct-q4_k_s.gguf new file mode 100644 index 0000000..813ce68 --- /dev/null +++ b/granite-3.2-2b-instruct-q4_k_s.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a6fcef1ccfbce46e1cee36de12b32ea24acab7de631db583dec49b31bab53247 +size 1477499744 diff --git a/granite-3.2-2b-instruct-q5_0.gguf b/granite-3.2-2b-instruct-q5_0.gguf new file mode 100644 index 0000000..7c49a08 --- /dev/null +++ b/granite-3.2-2b-instruct-q5_0.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1182b9a4271a690b0f212e58aaba08119e098589c35456d0606a013bdd18c2a7 +size 1744099360 diff --git a/granite-3.2-2b-instruct-q5_1.gguf b/granite-3.2-2b-instruct-q5_1.gguf new file mode 100644 index 0000000..51f7c2a --- /dev/null +++ b/granite-3.2-2b-instruct-q5_1.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ef19b8165a648d67048f38378ddae0a118a2366f5ec269f1d5e7fa0a6694c167 +size 1902434720 diff --git a/granite-3.2-2b-instruct-q5_k_m.gguf b/granite-3.2-2b-instruct-q5_k_m.gguf new file mode 100644 index 0000000..3a717dd --- /dev/null +++ b/granite-3.2-2b-instruct-q5_k_m.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:73f92edcbb118e290a6591f7ccde19ddc321f8e868cde08e2c55ba9f517e2be0 +size 1809275744 diff --git a/granite-3.2-2b-instruct-q5_k_s.gguf b/granite-3.2-2b-instruct-q5_k_s.gguf new file mode 100644 index 0000000..628e73f --- /dev/null +++ b/granite-3.2-2b-instruct-q5_k_s.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5e8bef6f1bcb2907e91b6d2a756085b80515d4b01cfc5aa9e075e47413b9652c +size 1780865888 diff --git a/granite-3.2-2b-instruct-q6_k_m.gguf b/granite-3.2-2b-instruct-q6_k_m.gguf new file mode 100644 index 0000000..4f57d4f --- /dev/null +++ b/granite-3.2-2b-instruct-q6_k_m.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8d2d8ad5a5d988ec13d5fe7ad9d9eb5567bc9143cf48cd00a7bc9035cc9a9a8f +size 2080562016 diff --git a/granite-3.2-2b-instruct-q8_0.gguf b/granite-3.2-2b-instruct-q8_0.gguf new file mode 100644 index 0000000..4e8d79e --- /dev/null +++ b/granite-3.2-2b-instruct-q8_0.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d9fa2635f1d1697ee747db6c215465e463d3cbb9fa52414bb8267b5549f02fe0 +size 2694111200 diff --git a/granite-3.2-2b-instruct-tq1_0.gguf b/granite-3.2-2b-instruct-tq1_0.gguf new file mode 100644 index 0000000..8042763 --- /dev/null +++ b/granite-3.2-2b-instruct-tq1_0.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:92faa35d83ac03595f04aed8b0d367bc1ea030d30167a17a0d26edc4f6b5d48c +size 598137440 diff --git a/granite-3.2-2b-instruct-tq2_0.gguf b/granite-3.2-2b-instruct-tq2_0.gguf new file mode 100644 index 0000000..042f1d7 --- /dev/null +++ b/granite-3.2-2b-instruct-tq2_0.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1e21fa99dd448ccf59908ae3a60ae4adc32ef23a2fa9bc2193963323e90f0e25 +size 712170080 diff --git a/granite-3.2-2b-instruct.imatrix b/granite-3.2-2b-instruct.imatrix new file mode 100644 index 0000000..04f63b0 --- /dev/null +++ b/granite-3.2-2b-instruct.imatrix @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:381ecc0cd03fbbb51284952e0c0654cf5d3ac65d98f6081f26cdacffc55c1db4 +size 3286098