初始化项目，由ModelHub XC社区提供模型

Model: Mungert/granite-3.2-8b-instruct-GGUF Source: Original Platform
2026-07-23 11:26:11 +08:00
commit 484e58fad6
38 changed files with 668 additions and 0 deletions
--- a/.gitattributes
+++ b/.gitattributes
@@ -0,0 +1,87 @@
 *.7z filter=lfs diff=lfs merge=lfs -text
 *.arrow filter=lfs diff=lfs merge=lfs -text
 *.bin filter=lfs diff=lfs merge=lfs -text
 *.bz2 filter=lfs diff=lfs merge=lfs -text
 *.ckpt filter=lfs diff=lfs merge=lfs -text
 *.ftz filter=lfs diff=lfs merge=lfs -text
 *.gz filter=lfs diff=lfs merge=lfs -text
 *.h5 filter=lfs diff=lfs merge=lfs -text
 *.joblib filter=lfs diff=lfs merge=lfs -text
 *.lfs.* filter=lfs diff=lfs merge=lfs -text
 *.mlmodel filter=lfs diff=lfs merge=lfs -text
 *.model filter=lfs diff=lfs merge=lfs -text
 *.msgpack filter=lfs diff=lfs merge=lfs -text
 *.npy filter=lfs diff=lfs merge=lfs -text
 *.npz filter=lfs diff=lfs merge=lfs -text
 *.onnx filter=lfs diff=lfs merge=lfs -text
 *.ot filter=lfs diff=lfs merge=lfs -text
 *.parquet filter=lfs diff=lfs merge=lfs -text
 *.pb filter=lfs diff=lfs merge=lfs -text
 *.pickle filter=lfs diff=lfs merge=lfs -text
 *.pkl filter=lfs diff=lfs merge=lfs -text
 *.pt filter=lfs diff=lfs merge=lfs -text
 *.pth filter=lfs diff=lfs merge=lfs -text
 *.rar filter=lfs diff=lfs merge=lfs -text
 *.safetensors filter=lfs diff=lfs merge=lfs -text
 saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.tar.* filter=lfs diff=lfs merge=lfs -text
 *.tar filter=lfs diff=lfs merge=lfs -text
 *.tflite filter=lfs diff=lfs merge=lfs -text
 *.tgz filter=lfs diff=lfs merge=lfs -text
 *.wasm filter=lfs diff=lfs merge=lfs -text
 *.xz filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
 granite-3.2-8b-instruct-iq3_m.gguf filter=lfs diff=lfs merge=lfs -text
 granite-3.2-8b-instruct-q8_0.gguf filter=lfs diff=lfs merge=lfs -text
 granite-3.2-8b-instruct-iq4_xs.gguf filter=lfs diff=lfs merge=lfs -text
 granite-3.2-8b-instruct-iq2_m.gguf filter=lfs diff=lfs merge=lfs -text
 granite-3.2-8b-instruct-q2_k_l.gguf filter=lfs diff=lfs merge=lfs -text
 granite-3.2-8b-instruct-iq2_xxs.gguf filter=lfs diff=lfs merge=lfs -text
 granite-3.2-8b-instruct.imatrix filter=lfs diff=lfs merge=lfs -text
 granite-3.2-8b-instruct-tq1_0.gguf filter=lfs diff=lfs merge=lfs -text
 granite-3.2-8b-instruct-q5_1.gguf filter=lfs diff=lfs merge=lfs -text
 granite-3.2-8b-instruct-q6_k_l.gguf filter=lfs diff=lfs merge=lfs -text
 granite-3.2-8b-instruct-iq3_s.gguf filter=lfs diff=lfs merge=lfs -text
 granite-3.2-8b-instruct-q4_k_m.gguf filter=lfs diff=lfs merge=lfs -text
 granite-3.2-8b-instruct-iq2_s.gguf filter=lfs diff=lfs merge=lfs -text
 granite-3.2-8b-instruct-q4_0_l.gguf filter=lfs diff=lfs merge=lfs -text
 granite-3.2-8b-instruct-bf16-q8_0.gguf filter=lfs diff=lfs merge=lfs -text
 granite-3.2-8b-instruct-q4_1_l.gguf filter=lfs diff=lfs merge=lfs -text
 granite-3.2-8b-instruct-q4_k_s.gguf filter=lfs diff=lfs merge=lfs -text
 granite-3.2-8b-instruct-iq3_xs.gguf filter=lfs diff=lfs merge=lfs -text
 granite-3.2-8b-instruct-q3_k_l.gguf filter=lfs diff=lfs merge=lfs -text
 granite-3.2-8b-instruct-q6_k_m.gguf filter=lfs diff=lfs merge=lfs -text
 granite-3.2-8b-instruct-q5_k_m.gguf filter=lfs diff=lfs merge=lfs -text
 granite-3.2-8b-instruct-f16-q6_k.gguf filter=lfs diff=lfs merge=lfs -text
 granite-3.2-8b-instruct-q4_0.gguf filter=lfs diff=lfs merge=lfs -text
 granite-3.2-8b-instruct-bf16-q4_k.gguf filter=lfs diff=lfs merge=lfs -text
 granite-3.2-8b-instruct-q2_k_s.gguf filter=lfs diff=lfs merge=lfs -text
 granite-3.2-8b-instruct-q5_k_l.gguf filter=lfs diff=lfs merge=lfs -text
 granite-3.2-8b-instruct-f16-q8_0.gguf filter=lfs diff=lfs merge=lfs -text
 granite-3.2-8b-instruct-f16.gguf filter=lfs diff=lfs merge=lfs -text
 granite-3.2-8b-instruct-bf16.gguf filter=lfs diff=lfs merge=lfs -text
 granite-3.2-8b-instruct-q4_1.gguf filter=lfs diff=lfs merge=lfs -text
 granite-3.2-8b-instruct-q3_k_m.gguf filter=lfs diff=lfs merge=lfs -text
 granite-3.2-8b-instruct-q5_0_l.gguf filter=lfs diff=lfs merge=lfs -text
 granite-3.2-8b-instruct-q5_1_l.gguf filter=lfs diff=lfs merge=lfs -text
 granite-3.2-8b-instruct-q4_k_l.gguf filter=lfs diff=lfs merge=lfs -text
 granite-3.2-8b-instruct-iq2_xs.gguf filter=lfs diff=lfs merge=lfs -text
 granite-3.2-8b-instruct-f16-q4_k.gguf filter=lfs diff=lfs merge=lfs -text
 granite-3.2-8b-instruct-tq2_0.gguf filter=lfs diff=lfs merge=lfs -text
 granite-3.2-8b-instruct-iq1_s.gguf filter=lfs diff=lfs merge=lfs -text
 granite-3.2-8b-instruct-iq1_m.gguf filter=lfs diff=lfs merge=lfs -text
 granite-3.2-8b-instruct-q5_0.gguf filter=lfs diff=lfs merge=lfs -text
 granite-3.2-8b-instruct-iq3_xxs.gguf filter=lfs diff=lfs merge=lfs -text
 granite-3.2-8b-instruct-bf16-q6_k.gguf filter=lfs diff=lfs merge=lfs -text
 granite-3.2-8b-instruct-q5_k_s.gguf filter=lfs diff=lfs merge=lfs -text
 granite-3.2-8b-instruct-iq4_nl.gguf filter=lfs diff=lfs merge=lfs -text
 granite-3.2-8b-instruct-q3_k_s.gguf filter=lfs diff=lfs merge=lfs -text
 granite-3.2-8b-instruct-f16_q8_0.gguf filter=lfs diff=lfs merge=lfs -text
 granite-3.2-8b-instruct-bf16_q8_0.gguf filter=lfs diff=lfs merge=lfs -text
 granite-3.2-8b-instruct-f16_q6_k.gguf filter=lfs diff=lfs merge=lfs -text
 granite-3.2-8b-instruct-bf16_q6_k.gguf filter=lfs diff=lfs merge=lfs -text
 granite-3.2-8b-instruct-f16_q4_k.gguf filter=lfs diff=lfs merge=lfs -text
 granite-3.2-8b-instruct-bf16_q4_k.gguf filter=lfs diff=lfs merge=lfs -text
 granite-3.2-8b-instruct-q2_k_m.gguf filter=lfs diff=lfs merge=lfs -text
--- a/README.md
+++ b/README.md
@@ -0,0 +1,473 @@
 ---
 pipeline_tag: text-generation
 inference: false
 license: apache-2.0
 library_name: transformers
 tags:
 - language
 - granite-3.2
 base_model:
 - ibm-granite/granite-3.1-8b-instruct
 new_version: ibm-granite/granite-3.3-8b-instruct
 ---
 # <span style="color: #7FFF7F;">granite-3.2-8b-instruct GGUF Models</span>
 ## <span style="color: #7F7FFF;">Model Generation Details</span>
 This model was generated using [llama.cpp](https://github.com/ggerganov/llama.cpp) at commit [`5dd942de`](https://github.com/ggerganov/llama.cpp/commit/5dd942de5922a22ec8446a4ad2203738dbcb9389).
 ---
 ## <span style="color: #7FFF7F;">Quantization Beyond the IMatrix</span>
 I've been experimenting with a new quantization approach that selectively elevates the precision of key layers beyond what the default IMatrix configuration provides.
 In my testing, standard IMatrix quantization underperforms at lower bit depths, especially with Mixture of Experts (MoE) models. To address this, I'm using the `--tensor-type` option in `llama.cpp` to manually "bump" important layers to higher precision. You can see the implementation here:  
 👉 [Layer bumping with llama.cpp](https://github.com/Mungert69/GGUFModelBuilder/blob/main/model-converter/tensor_list_builder.py)
 While this does increase model file size, it significantly improves precision for a given quantization level.
 ### **I'd love your feedback—have you tried this? How does it perform for you?**
 ---
 <a href="https://readyforquantum.com/huggingface_gguf_selection_guide.html" style="color: #7FFF7F;">
  Click here to get info on choosing the right GGUF model format
 </a>
 ---
 <!--Begin Original Model Card-->
 # Granite-3.2-8B-Instruct
 **Model Summary:**
 Granite-3.2-8B-Instruct is an 8-billion-parameter, long-context AI model fine-tuned for thinking capabilities. Built on top of [Granite-3.1-8B-Instruct](https://huggingface.co/ibm-granite/granite-3.1-8b-instruct), it has been trained using a mix of permissively licensed open-source datasets and internally generated synthetic data designed for reasoning tasks. The model allows controllability of its thinking capability, ensuring it is applied only when required.
 - **Developers:** Granite Team, IBM
 - **Website**: [Granite Docs](https://www.ibm.com/granite/docs/)
 - **Release Date**: February 26th, 2025
 - **License:** [Apache 2.0](https://www.apache.org/licenses/LICENSE-2.0)
 **Supported Languages:** 
 English, German, Spanish, French, Japanese, Portuguese, Arabic, Czech, Italian, Korean, Dutch, and Chinese. However, users may finetune this Granite model for languages beyond these 12 languages.
 **Intended Use:** 
 This model is designed to handle general instruction-following tasks and can be integrated into AI assistants across various domains, including business applications.
 **Capabilities**
 * **Thinking**
 * Summarization
 * Text classification
 * Text extraction
 * Question-answering
 * Retrieval Augmented Generation (RAG)
 * Code related tasks
 * Function-calling tasks
 * Multilingual dialog use cases
 * Long-context tasks including long document/meeting summarization, long document QA, etc.
 **Generation:** 
 This is a simple example of how to use Granite-3.2-8B-Instruct model.
 Install the following libraries:
 ```shell
 pip install torch torchvision torchaudio
 pip install accelerate
 pip install transformers
 ```
 Then, copy the snippet from the section that is relevant for your use case.
 ```python
 from transformers import AutoModelForCausalLM, AutoTokenizer, set_seed
 import torch
 model_path="ibm-granite/granite-3.2-8b-instruct"
 device="cuda"
 model = AutoModelForCausalLM.from_pretrained(
        model_path,
        device_map=device,
        torch_dtype=torch.bfloat16,
    )
 tokenizer = AutoTokenizer.from_pretrained(
        model_path
 )
 conv = [{"role": "user", "content":"You have 10 liters of a 30% acid solution. How many liters of a 70% acid solution must be added to achieve a 50% acid mixture?"}]
 input_ids = tokenizer.apply_chat_template(conv, return_tensors="pt", thinking=True, return_dict=True, add_generation_prompt=True).to(device)
 set_seed(42)
 output = model.generate(
    **input_ids,
    max_new_tokens=8192,
 )
 prediction = tokenizer.decode(output[0, input_ids["input_ids"].shape[1]:], skip_special_tokens=True)
 print(prediction)
 ```
 **Example Outputs**
 - thinking=True
 ```md
 Here is my thought process:
 1. We know we start with 10 liters of a 30% acid solution, so there's 3 liters of pure acid in it (10 liters * 0.3).
 2. We need to determine how much of the 70% acid solution to add such that the final mixture is 50% acid.
 3. Let's say x liters of the 70% acid solution are added. This will contribute 0.7x liters of pure acid.
 4. The total volume of the new mixture will be 10 + x liters, and the total amount of pure acid will be 3 + 0.7x liters.
 5. We need to set up an equation where (3 + 0.7x) / (10 + x) = 0.5 (since we want a 50% acid mixture).
 6. Solving this equation for x will give us the volume of 70% solution needed.
 Here is my response:
 To solve this, we need to set up an equation based on the concentration of acid before and after mixing. 
 1. **Initial Acid Content**: 
   - We start with 10 liters of a 30% acid solution. 
   - Pure acid in this = 10 * 0.3 = 3 liters.
 2. **Variable for Addition**:
   - Let x be the liters of 70% acid solution to be added.
   - Pure acid in x liters of 70% solution = 0.7x liters.
 3. **Final Mixture Concentration**:
   - Total volume of mixture = 10 + x liters.
   - Total pure acid in the mixture = 3 (from initial solution) + 0.7x (from added solution).
 4. **Setting up the equation for 50% acid concentration**:
   - We require that the ratio of pure acid to total volume is 0.5 (50%).
   - Therefore, the equation is:
   \[
   \frac{3 + 0.7x}{10 + x} = 0.5
   \]
 5. **Solving the equation**:
   Multiply both sides by (10 + x) to clear the fraction:
   \[
   3 + 0.7x = 0.5(10 + x)
   \]
   Expanding the right side:
   \[
   3 + 0.7x = 5 + 0.5x
   \]
   Subtract 0.5x from both sides:
   \[
   3 + 0.2x = 5
   \]
   Subtract 3 from both sides:
   \[
   0.2x = 2
   \]
   Divide by 0.2:
   \[
   x = 10
   \]
 So, 10 liters of the 70% acid solution must be added to the initial 10 liters of 30% solution to get a 50% acid mixture. 
 Let's verify: 
 - Initial acid: 3 liters
 - Acid from added solution: 0.7 * 10 = 7 liters
 - Total acid: 3 + 7 = 10 liters
 - Total volume: 10 + 10 = 20 liters
 - Acid percentage: \(\frac{10}{20} = 0.5 = 50\%\)
 This confirms our solution. 
 Final answer: 10 liters of 70% acid solution should be added.
 ```
 - thinking=False
 ```md
 Let's denote the amount of 70% acid solution to be added as x liters.
 We start with 10 liters of a 30% acid solution, which means there are 0.3 * 10 = 3 liters of pure acid in it.
 We want to end up with a 50% acid solution after adding x liters of 70% acid solution. The total volume of the mixture will be 10 + x liters, and we want half of this volume to be pure acid.
 So, the amount of pure acid in the final mixture should be 0.5 * (10 + x) liters.
 The amount of pure acid in the final mixture comes from two sources: the original 30% solution and the added 70% solution. So, we can set up the following equation:
 3 (from the original solution) + 0.7x (from the added solution) = 0.5 * (10 + x)
 Now, let's solve for x:
 3 + 0.7x = 5 + 0.5x
 0.7x - 0.5x = 5 - 3
 0.2x = 2
 x = 2 / 0.2
 x = 10
 So, you need to add 10 liters of a 70% acid solution to the 10 liters of a 30% acid solution to get a 50% acid mixture.
 ```
 **Evaluation Results:**
 <table>
 <thead>
  <tr>
    <th style="text-align:left; background-color: #001d6c; color: white;">Models</th>
    <th style="text-align:center; background-color: #001d6c; color: white;">ArenaHard</th>
    <th style="text-align:center; background-color: #001d6c; color: white;">Alpaca-Eval-2</th>
    <th style="text-align:center; background-color: #001d6c; color: white;">MMLU</th>
    <th style="text-align:center; background-color: #001d6c; color: white;">PopQA</th>
    <th style="text-align:center; background-color: #001d6c; color: white;">TruthfulQA</th>
    <th style="text-align:center; background-color: #001d6c; color: white;">BigBenchHard</th>
    <th style="text-align:center; background-color: #001d6c; color: white;">DROP</th>
    <th style="text-align:center; background-color: #001d6c; color: white;">GSM8K</th>
    <th style="text-align:center; background-color: #001d6c; color: white;">HumanEval</th>
   <th style="text-align:center; background-color: #001d6c; color: white;">HumanEval+</th>
  <th style="text-align:center; background-color: #001d6c; color: white;">IFEval</th>
    <th style="text-align:center; background-color: #001d6c; color: white;">AttaQ</th>
  </tr></thead>
  <tbody>
  <tr>
    <td style="text-align:left; background-color: #DAE8FF; color: black;">Llama-3.1-8B-Instruct</td>
    <td style="text-align:center; background-color: #DAE8FF; color: black;">36.43</td>
    <td style="text-align:center; background-color: #DAE8FF; color: black;">27.22</td>
    <td style="text-align:center; background-color: #DAE8FF; color: black;">69.15</td>
    <td style="text-align:center; background-color: #DAE8FF; color: black;">28.79</td>
    <td style="text-align:center; background-color: #DAE8FF; color: black;">52.79</td>
    <td style="text-align:center; background-color: #DAE8FF; color: black;">72.66</td>
    <td style="text-align:center; background-color: #DAE8FF; color: black;">61.48</td>
    <td style="text-align:center; background-color: #DAE8FF; color: black;">83.24</td>
    <td style="text-align:center; background-color: #DAE8FF; color: black;">85.32</td>
    <td style="text-align:center; background-color: #DAE8FF; color: black;">80.15</td>
    <td style="text-align:center; background-color: #DAE8FF; color: black;">79.10</td>
    <td style="text-align:center; background-color: #DAE8FF; color: black;">83.43</td>
  </tr>
  <tr>
    <td style="text-align:left; background-color: #DAE8FF; color: black;">DeepSeek-R1-Distill-Llama-8B</td>
    <td style="text-align:center; background-color: #DAE8FF; color: black;">17.17</td>
    <td style="text-align:center; background-color: #DAE8FF; color: black;">21.85</td>
    <td style="text-align:center; background-color: #DAE8FF; color: black;">45.80</td>
    <td style="text-align:center; background-color: #DAE8FF; color: black;">13.25</td>
    <td style="text-align:center; background-color: #DAE8FF; color: black;">47.43</td>
    <td style="text-align:center; background-color: #DAE8FF; color: black;">65.71</td>
    <td style="text-align:center; background-color: #DAE8FF; color: black;">44.46</td>
    <td style="text-align:center; background-color: #DAE8FF; color: black;">72.18</td>
    <td style="text-align:center; background-color: #DAE8FF; color: black;">67.54</td>
    <td style="text-align:center; background-color: #DAE8FF; color: black;">62.91</td>
    <td style="text-align:center; background-color: #DAE8FF; color: black;">66.50</td>
    <td style="text-align:center; background-color: #DAE8FF; color: black;">42.87</td>
  </tr>
  <tr>
    <td style="text-align:left; background-color: #DAE8FF; color: black;">Qwen-2.5-7B-Instruct</td>
    <td style="text-align:center; background-color: #DAE8FF; color: black;">25.44</td>
    <td style="text-align:center; background-color: #DAE8FF; color: black;">30.34</td>
    <td style="text-align:center; background-color: #DAE8FF; color: black;">74.30</td>
    <td style="text-align:center; background-color: #DAE8FF; color: black;">18.12</td>
    <td style="text-align:center; background-color: #DAE8FF; color: black;">63.06</td>
    <td style="text-align:center; background-color: #DAE8FF; color: black;">70.40</td>
    <td style="text-align:center; background-color: #DAE8FF; color: black;">54.71</td>
    <td style="text-align:center; background-color: #DAE8FF; color: black;">84.46</td>
    <td style="text-align:center; background-color: #DAE8FF; color: black;">93.35</td>
    <td style="text-align:center; background-color: #DAE8FF; color: black;">89.91</td>
    <td style="text-align:center; background-color: #DAE8FF; color: black;">74.90</td>
    <td style="text-align:center; background-color: #DAE8FF; color: black;">81.90</td>
  </tr>
  <tr>
    <td style="text-align:left; background-color: #DAE8FF; color: black;">DeepSeek-R1-Distill-Qwen-7B</td>
    <td style="text-align:center; background-color: #DAE8FF; color: black;">10.36</td>
    <td style="text-align:center; background-color: #DAE8FF; color: black;">15.35</td>
    <td style="text-align:center; background-color: #DAE8FF; color: black;">50.72</td>
    <td style="text-align:center; background-color: #DAE8FF; color: black;">9.94</td>
    <td style="text-align:center; background-color: #DAE8FF; color: black;">47.14</td>
    <td style="text-align:center; background-color: #DAE8FF; color: black;">65.04</td>
    <td style="text-align:center; background-color: #DAE8FF; color: black;">42.76</td>
    <td style="text-align:center; background-color: #DAE8FF; color: black;">78.47</td>
    <td style="text-align:center; background-color: #DAE8FF; color: black;">79.89</td>
    <td style="text-align:center; background-color: #DAE8FF; color: black;">78.43</td>
    <td style="text-align:center; background-color: #DAE8FF; color: black;">59.10</td>
    <td style="text-align:center; background-color: #DAE8FF; color: black;">42.45</td>
  </tr>
  <tr>
    <td style="text-align:left; background-color: #DAE8FF; color: black;">Granite-3.1-8B-Instruct</td>
    <td style="text-align:center; background-color: #DAE8FF; color: black;">37.58</td>
    <td style="text-align:center; background-color: #DAE8FF; color: black;">30.34</td>
    <td style="text-align:center; background-color: #DAE8FF; color: black;">66.77</td>
    <td style="text-align:center; background-color: #DAE8FF; color: black;">28.7</td>
    <td style="text-align:center; background-color: #DAE8FF; color: black;">65.84</td>
    <td style="text-align:center; background-color: #DAE8FF; color: black;">68.55</td>
    <td style="text-align:center; background-color: #DAE8FF; color: black;">50.78</td>
    <td style="text-align:center; background-color: #DAE8FF; color: black;">79.15</td>
    <td style="text-align:center; background-color: #DAE8FF; color: black;">89.63</td>
    <td style="text-align:center; background-color: #DAE8FF; color: black;">85.79</td>
    <td style="text-align:center; background-color: #DAE8FF; color: black;">73.20</td>
    <td style="text-align:center; background-color: #DAE8FF; color: black;">85.73</td>
  </tr>
 <tr>
    <td style="text-align:left; background-color: #DAE8FF; color: black;">Granite-3.1-2B-Instruct</td>
    <td style="text-align:center; background-color: #DAE8FF; color: black;">23.3</td>
    <td style="text-align:center; background-color: #DAE8FF; color: black;">27.17</td>
    <td style="text-align:center; background-color: #DAE8FF; color: black;">57.11</td> 
    <td style="text-align:center; background-color: #DAE8FF; color: black;">20.55</td>
    <td style="text-align:center; background-color: #DAE8FF; color: black;">59.79</td>
    <td style="text-align:center; background-color: #DAE8FF; color: black;">54.46</td>
    <td style="text-align:center; background-color: #DAE8FF; color: black;">18.68</td>
    <td style="text-align:center; background-color: #DAE8FF; color: black;">67.55</td>
    <td style="text-align:center; background-color: #DAE8FF; color: black;">79.45</td>
    <td style="text-align:center; background-color: #DAE8FF; color: black;">75.26</td>
    <td style="text-align:center; background-color: #DAE8FF; color: black;">63.59</td>
    <td style="text-align:center; background-color: #DAE8FF; color: black;">84.7</td>
  </tr>
  <tr>
      <td style="text-align:left; background-color: #DAE8FF; color: black;">Granite-3.2-2B-Instruct</td>
    <td style="text-align:center; background-color: #DAE8FF; color: black;">24.86</td>
    <td style="text-align:center; background-color: #DAE8FF; color: black;">34.51</td>
    <td style="text-align:center; background-color: #DAE8FF; color: black;">57.18</td>
    <td style="text-align:center; background-color: #DAE8FF; color: black;">20.56</td>
    <td style="text-align:center; background-color: #DAE8FF; color: black;">59.8</td>
    <td style="text-align:center; background-color: #DAE8FF; color: black;">52.27</td>
    <td style="text-align:center; background-color: #DAE8FF; color: black;">21.12</td>
    <td style="text-align:center; background-color: #DAE8FF; color: black;">67.02</td>
    <td style="text-align:center; background-color: #DAE8FF; color: black;">80.13</td>
    <td style="text-align:center; background-color: #DAE8FF; color: black;">73.39</td>
    <td style="text-align:center; background-color: #DAE8FF; color: black;">61.55</td>
    <td style="text-align:center; background-color: #DAE8FF; color: black;">83.23</td>
  </tr> 
 <tr>
      <td style="text-align:left; background-color: #DAE8FF; color: black;"><b>Granite-3.2-8B-Instruct</b></td>
    <td style="text-align:center; background-color: #DAE8FF; color: black;">55.25</td>
    <td style="text-align:center; background-color: #DAE8FF; color: black;">61.19</td>
   <td style="text-align:center; background-color: #DAE8FF; color: black;">66.79</td>
    <td style="text-align:center; background-color: #DAE8FF; color: black;">28.04</td>
    <td style="text-align:center; background-color: #DAE8FF; color: black;">66.92</td>
    <td style="text-align:center; background-color: #DAE8FF; color: black;">64.77</td>
    <td style="text-align:center; background-color: #DAE8FF; color: black;">50.95</td>
    <td style="text-align:center; background-color: #DAE8FF; color: black;">81.65</td>
    <td style="text-align:center; background-color: #DAE8FF; color: black;">89.35</td>
    <td style="text-align:center; background-color: #DAE8FF; color: black;">85.72</td>
    <td style="text-align:center; background-color: #DAE8FF; color: black;">74.31</td>
    <td style="text-align:center; background-color: #DAE8FF; color: black;">85.42</td>
  </tr>
 </tbody></table>
 **Training Data:** 
 Overall, our training data is largely comprised of two key sources: (1) publicly available datasets with permissive license, (2) internal synthetically generated data targeted to enhance reasoning capabilites. 
 <!-- A detailed attribution of datasets can be found in [Granite 3.2 Technical Report (coming soon)](#), and [Accompanying Author List](https://github.com/ibm-granite/granite-3.0-language-models/blob/main/author-ack.pdf). -->
 **Infrastructure:**
 We train Granite-3.2-8B-Instruct using IBM's super computing cluster, Blue Vela, which is outfitted with NVIDIA H100 GPUs. This cluster provides a scalable and efficient infrastructure for training our models over thousands of GPUs.
 **Ethical Considerations and Limitations:** 
 Granite-3.2-8B-Instruct builds upon Granite-3.1-8B-Instruct, leveraging both permissively licensed open-source and select proprietary data for enhanced performance. Since it inherits its foundation from the previous model, all ethical considerations and limitations applicable to [Granite-3.1-8B-Instruct](https://huggingface.co/ibm-granite/granite-3.1-8b-instruct) remain relevant.
 **Resources**
 - ⭐️ Learn about the latest updates with Granite: https://www.ibm.com/granite
 - 📄 Get started with tutorials, best practices, and prompt engineering advice: https://www.ibm.com/granite/docs/
 - 💡 Learn about the latest Granite learning resources: https://ibm.biz/granite-learning-resources
 <!-- ## Citation
 ```
@misc{granite-models,
  author = {author 1, author2, ...},
  title = {},
  journal = {},
  volume = {},
  year = {2024},
  url = {https://arxiv.org/abs/0000.00000},
 }
 ``` -->
 <!--End Original Model Card-->
 ---
 # <span id="testllm" style="color: #7F7FFF;">🚀 If you find these models useful</span>
 Help me test my **AI-Powered Quantum Network Monitor Assistant** with **quantum-ready security checks**:  
 👉 [Quantum Network Monitor](https://readyforquantum.com/?assistant=open&utm_source=huggingface&utm_medium=referral&utm_campaign=huggingface_repo_readme)  
 The full Open Source Code for the Quantum Network Monitor Service available at my github repos ( repos with NetworkMonitor in the name) : [Source Code Quantum Network Monitor](https://github.com/Mungert69). You will also find the code I use to quantize the models if you want to do it yourself [GGUFModelBuilder](https://github.com/Mungert69/GGUFModelBuilder)
 💬 **How to test**:  
 Choose an **AI assistant type**:  
   - `TurboLLM` (GPT-4.1-mini)  
   - `HugLLM` (Hugginface Open-source models)  
   - `TestLLM` (Experimental CPU-only)  
 ### **What I’m Testing**  
 I’m pushing the limits of **small open-source models for AI network monitoring**, specifically:  
 - **Function calling** against live network services  
 - **How small can a model go** while still handling:  
  - Automated **Nmap security scans**  
  - **Quantum-readiness checks**  
  - **Network Monitoring tasks**  
 🟡 **TestLLM** – Current experimental model (llama.cpp on 2 CPU threads on huggingface docker space):  
 - ✅ **Zero-configuration setup**  
 - ⏳ 30s load time (slow inference but **no API costs**) . No token limited as the cost is low.
 - 🔧 **Help wanted!** If you’re into **edge-device AI**, let’s collaborate!  
 ### **Other Assistants**  
 🟢 **TurboLLM** – Uses **gpt-4.1-mini** :
 - **It performs very well but unfortunatly OpenAI charges per token. For this reason tokens usage is limited. 
 - **Create custom cmd processors to run .net code on Quantum Network Monitor Agents**
 - **Real-time network diagnostics and monitoring**
 - **Security Audits**
 - **Penetration testing** (Nmap/Metasploit)  
 🔵 **HugLLM** – Latest Open-source models:  
 - 🌐 Runs on Hugging Face Inference API. Performs pretty well using the lastest models hosted on Novita.
 ### 💡 **Example commands you could test**:  
 1. `"Give me info on my websites SSL certificate"`  
 2. `"Check if my server is using quantum safe encyption for communication"`  
 3. `"Run a comprehensive security audit on my server"`
 4. '"Create a cmd processor to .. (what ever you want)" Note you need to install a [Quantum Network Monitor Agent](https://readyforquantum.com/Download/?utm_source=huggingface&utm_medium=referral&utm_campaign=huggingface_repo_readme) to run the .net code on. This is a very flexible and powerful feature. Use with caution!
 ### Final Word
 I fund the servers used to create these model files, run the Quantum Network Monitor service, and pay for inference from Novita and OpenAI—all out of my own pocket. All the code behind the model creation and the Quantum Network Monitor project is [open source](https://github.com/Mungert69). Feel free to use whatever you find helpful.
 If you appreciate the work, please consider [buying me a coffee](https://www.buymeacoffee.com/mahadeva) ☕. Your support helps cover service costs and allows me to raise token limits for everyone.
 I'm also open to job opportunities or sponsorship.
 Thank you! 😊
--- a/granite-3.2-8b-instruct-bf16-q4_k.gguf
+++ b/granite-3.2-8b-instruct-bf16-q4_k.gguf
@@ -0,0 +1,3 @@
 version https://git-lfs.github.com/spec/v1
 oid sha256:649c1f5870a154a98e14e3e7aeb936e7367cc7672c0ec4cff4c322c47369ca6e
 size 5180376800
--- a/granite-3.2-8b-instruct-bf16-q6_k.gguf
+++ b/granite-3.2-8b-instruct-bf16-q6_k.gguf
@@ -0,0 +1,3 @@
 version https://git-lfs.github.com/spec/v1
 oid sha256:14ff6575a660ce5f83323e137f0b6b577701ed6ebe9c2c0e3a9f4a76c65070ec
 size 6942967520
--- a/granite-3.2-8b-instruct-bf16-q8_0.gguf
+++ b/granite-3.2-8b-instruct-bf16-q8_0.gguf
@@ -0,0 +1,3 @@
 version https://git-lfs.github.com/spec/v1
 oid sha256:ff67ef5ceafc133ea52d8007fc56761af4a32d999af55755f2f22530559fa2ec
 size 8873002432
--- a/granite-3.2-8b-instruct-bf16.gguf
+++ b/granite-3.2-8b-instruct-bf16.gguf
@@ -0,0 +1,3 @@
 version https://git-lfs.github.com/spec/v1
 oid sha256:9f64c7b3d9dd694642d4cf2caa40898e4f27cffccbe5ba06baf2e9fe4c16d240
 size 16344106592
--- a/granite-3.2-8b-instruct-bf16_q8_0.gguf
+++ b/granite-3.2-8b-instruct-bf16_q8_0.gguf
@@ -0,0 +1,3 @@
 version https://git-lfs.github.com/spec/v1
 oid sha256:dced7245dc5f8dd4db694d1d80689e40640213741bffdada7964469f7e00ddd2
 size 11625514592
--- a/granite-3.2-8b-instruct-f16-q4_k.gguf
+++ b/granite-3.2-8b-instruct-f16-q4_k.gguf
@@ -0,0 +1,3 @@
 version https://git-lfs.github.com/spec/v1
 oid sha256:1aa28e7ce1dc76e6ebec6207c16e94aaa68edd09bc88b46d20ce8eed5f2fe454
 size 5180376800
--- a/granite-3.2-8b-instruct-f16-q6_k.gguf
+++ b/granite-3.2-8b-instruct-f16-q6_k.gguf
@@ -0,0 +1,3 @@
 version https://git-lfs.github.com/spec/v1
 oid sha256:ab85c382cb283b45b6d2570e55355380007b573dbcca92c8588e6dc60ee0f3dc
 size 6942967520
--- a/granite-3.2-8b-instruct-f16-q8_0.gguf
+++ b/granite-3.2-8b-instruct-f16-q8_0.gguf
@@ -0,0 +1,3 @@
 version https://git-lfs.github.com/spec/v1
 oid sha256:8808785957ecf7e85f0e897426ac45db5c3daacc29be7646b43b85df2747620b
 size 8873002432
--- a/granite-3.2-8b-instruct-f16_q8_0.gguf
+++ b/granite-3.2-8b-instruct-f16_q8_0.gguf
@@ -0,0 +1,3 @@
 version https://git-lfs.github.com/spec/v1
 oid sha256:f96be6cf02ae555568d4ef378c93e5bf07766ef761e33ecca2ab069448d801b0
 size 11625514592
--- a/granite-3.2-8b-instruct-iq2_m.gguf
+++ b/granite-3.2-8b-instruct-iq2_m.gguf
@@ -0,0 +1,3 @@
 version https://git-lfs.github.com/spec/v1
 oid sha256:221c5734644351017a7f69a611851a1f36d9464f1cf0f4ce6046c8878e3215be
 size 2935294112
--- a/granite-3.2-8b-instruct-iq2_s.gguf
+++ b/granite-3.2-8b-instruct-iq2_s.gguf
@@ -0,0 +1,3 @@
 version https://git-lfs.github.com/spec/v1
 oid sha256:7bd69d01b3ae392783f5ba614c6c322ea889cc9f709619c7ebab10766450d64d
 size 2788493472
--- a/granite-3.2-8b-instruct-iq2_xs.gguf
+++ b/granite-3.2-8b-instruct-iq2_xs.gguf
@@ -0,0 +1,3 @@
 version https://git-lfs.github.com/spec/v1
 oid sha256:4c176cbe3ccd775b8ec7d75ae14e825134077998eac2751823446c9eb473061c
 size 2689534112
--- a/granite-3.2-8b-instruct-iq2_xxs.gguf
+++ b/granite-3.2-8b-instruct-iq2_xxs.gguf
@@ -0,0 +1,3 @@
 version https://git-lfs.github.com/spec/v1
 oid sha256:c8af020185f5654609438b3c54d144c7ffb1d49a331dd2184ac3ded740fcb0c5
 size 2463434912
--- a/granite-3.2-8b-instruct-iq3_m.gguf
+++ b/granite-3.2-8b-instruct-iq3_m.gguf
@@ -0,0 +1,3 @@
 version https://git-lfs.github.com/spec/v1
 oid sha256:648cadc7a855909e4ae5db579e6ea1fee6f9331a26170104115f004ff7f66a93
 size 3823569056
--- a/granite-3.2-8b-instruct-iq3_s.gguf
+++ b/granite-3.2-8b-instruct-iq3_s.gguf
@@ -0,0 +1,3 @@
 version https://git-lfs.github.com/spec/v1
 oid sha256:cd72c194305ccc88dd89a04217b612d97e068ec4fdb153e7172b9015731a24c1
 size 3823569056
--- a/granite-3.2-8b-instruct-iq3_xs.gguf
+++ b/granite-3.2-8b-instruct-iq3_xs.gguf
@@ -0,0 +1,3 @@
 version https://git-lfs.github.com/spec/v1
 oid sha256:802ab16ebc2c148bfabad62bcbce9d9d29925727604fc0b13d81656ad4324dfa
 size 3424979104
--- a/granite-3.2-8b-instruct-iq3_xxs.gguf
+++ b/granite-3.2-8b-instruct-iq3_xxs.gguf
@@ -0,0 +1,3 @@
 version https://git-lfs.github.com/spec/v1
 oid sha256:17be3c08e331617a1d80c62d7008dbe99118971bd30c575e997cdd5ea6dc9a7c
 size 3323398304
--- a/granite-3.2-8b-instruct-iq4_nl.gguf
+++ b/granite-3.2-8b-instruct-iq4_nl.gguf
@@ -0,0 +1,3 @@
 version https://git-lfs.github.com/spec/v1
 oid sha256:d8531f8a73dfba9fc2c78cfe7e1d85652b7446939e5f62e2378a02822433bbc6
 size 4671868672
--- a/granite-3.2-8b-instruct-iq4_xs.gguf
+++ b/granite-3.2-8b-instruct-iq4_xs.gguf
@@ -0,0 +1,3 @@
 version https://git-lfs.github.com/spec/v1
 oid sha256:fe4f584e28f7c6a2a0364a386d43b0782eec3a6a075d22372cb9aba3a594d660
 size 4428074752
--- a/granite-3.2-8b-instruct-q2_k_m.gguf
+++ b/granite-3.2-8b-instruct-q2_k_m.gguf
@@ -0,0 +1,3 @@
 version https://git-lfs.github.com/spec/v1
 oid sha256:d7cb4e144bbe6311d137ce962bcfcf4911f66113da3de1cf1738c5f948977886
 size 3056144128
--- a/granite-3.2-8b-instruct-q2_k_s.gguf
+++ b/granite-3.2-8b-instruct-q2_k_s.gguf
@@ -0,0 +1,3 @@
 version https://git-lfs.github.com/spec/v1
 oid sha256:a9e0dd7f45df86312eac3c98a4938f360315a3b5d5f988dd5bf75c5ed624ece6
 size 3029403808
--- a/granite-3.2-8b-instruct-q3_k_m.gguf
+++ b/granite-3.2-8b-instruct-q3_k_m.gguf
@@ -0,0 +1,3 @@
 version https://git-lfs.github.com/spec/v1
 oid sha256:66807161a1a9435fbfb118fedb0c61e82657cb8b8ed9184adcfbe40cfa28ed86
 size 3965652736
--- a/granite-3.2-8b-instruct-q3_k_s.gguf
+++ b/granite-3.2-8b-instruct-q3_k_s.gguf
@@ -0,0 +1,3 @@
 version https://git-lfs.github.com/spec/v1
 oid sha256:c11b0c3fb397daf087c8f80e9c96a2683e213761f76a059df2568ae8ecb248b5
 size 3938912416
--- a/granite-3.2-8b-instruct-q4_0.gguf
+++ b/granite-3.2-8b-instruct-q4_0.gguf
@@ -0,0 +1,3 @@
 version https://git-lfs.github.com/spec/v1
 oid sha256:c66c533a6140e381499a80087ff7594dddf1c5423341bffa286dca788ca9cbd9
 size 4598989472
--- a/granite-3.2-8b-instruct-q4_1.gguf
+++ b/granite-3.2-8b-instruct-q4_1.gguf
@@ -0,0 +1,3 @@
 version https://git-lfs.github.com/spec/v1
 oid sha256:ffef9afa103c4403c7a59819e60d5debe25a99f3700eb9f2106f8df8dd89d970
 size 5109646752
--- a/granite-3.2-8b-instruct-q4_k_m.gguf
+++ b/granite-3.2-8b-instruct-q4_k_m.gguf
@@ -0,0 +1,3 @@
 version https://git-lfs.github.com/spec/v1
 oid sha256:ef649a47b886521beeb01d53d6a9d49e894b2e67eb54b85a54b322769ce701cc
 size 4899475200
--- a/granite-3.2-8b-instruct-q4_k_s.gguf
+++ b/granite-3.2-8b-instruct-q4_k_s.gguf
@@ -0,0 +1,3 @@
 version https://git-lfs.github.com/spec/v1
 oid sha256:6531aef31fba482f9597799ac721dfe0c389dd730f841eba7c4453f43fd2f1f4
 size 4730851072
--- a/granite-3.2-8b-instruct-q5_0.gguf
+++ b/granite-3.2-8b-instruct-q5_0.gguf
@@ -0,0 +1,3 @@
 version https://git-lfs.github.com/spec/v1
 oid sha256:e57b44d196c3ea822d4e044e5bb40921f14fb7cd6c2d5e7c9bf8c0e21c0e0720
 size 5620304032
--- a/granite-3.2-8b-instruct-q5_1.gguf
+++ b/granite-3.2-8b-instruct-q5_1.gguf
@@ -0,0 +1,3 @@
 version https://git-lfs.github.com/spec/v1
 oid sha256:8dd723b8ddb59851faa31f5bcdd277b3f73e920812e6e7570914ee1fea2260c9
 size 6130961312
--- a/granite-3.2-8b-instruct-q5_k_m.gguf
+++ b/granite-3.2-8b-instruct-q5_k_m.gguf
@@ -0,0 +1,3 @@
 version https://git-lfs.github.com/spec/v1
 oid sha256:f9467460fbf2817a6a0e8780ce7abb402ec0db5882a4b5a44e4a641f180ebc26
 size 5815275264
--- a/granite-3.2-8b-instruct-q5_k_s.gguf
+++ b/granite-3.2-8b-instruct-q5_k_s.gguf
@@ -0,0 +1,3 @@
 version https://git-lfs.github.com/spec/v1
 oid sha256:090e6d5df011e17b0a56f8be0e3118b689dc3fe88bc5a40f63a7e8d1376b6a33
 size 5725032192
--- a/granite-3.2-8b-instruct-q6_k_m.gguf
+++ b/granite-3.2-8b-instruct-q6_k_m.gguf
@@ -0,0 +1,3 @@
 version https://git-lfs.github.com/spec/v1
 oid sha256:ce4ca62461e6ccb8606ef086cf04d4c3c08a4050ad97ba9f0a6b675d2cafc32d
 size 6705450752
--- a/granite-3.2-8b-instruct-q8_0.gguf
+++ b/granite-3.2-8b-instruct-q8_0.gguf
@@ -0,0 +1,3 @@
 version https://git-lfs.github.com/spec/v1
 oid sha256:4703821e4387c3fcac69769af7e5d7ac3ef266a7eb27c1d3fde7f59025b40824
 size 8684247392
--- a/granite-3.2-8b-instruct-tq1_0.gguf
+++ b/granite-3.2-8b-instruct-tq1_0.gguf
@@ -0,0 +1,3 @@
 version https://git-lfs.github.com/spec/v1
 oid sha256:78727a742aa13c5cd74fd122a5d160830b9377a06a027ed05c21d9256d019795
 size 1849232960
--- a/granite-3.2-8b-instruct-tq2_0.gguf
+++ b/granite-3.2-8b-instruct-tq2_0.gguf
@@ -0,0 +1,3 @@
 version https://git-lfs.github.com/spec/v1
 oid sha256:33587d663c678c70b0e8ef1fef5e53821b791984da1f6ec5170cd3ee35dc675e
 size 2222788160
--- a/granite-3.2-8b-instruct.imatrix
+++ b/granite-3.2-8b-instruct.imatrix
@@ -0,0 +1,3 @@
 version https://git-lfs.github.com/spec/v1
 oid sha256:8fcf0fe90f2729f38e6982c1923e4b4fbf02e252e55d4d931c8e762647905ff5
 size 5989458