From 44f1eb55d7e9afac16f03ddb740a0d6d453be778 Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Mon, 13 Jul 2026 21:32:10 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: Tiiny/SmallThinker-21BA3B-Instruct-GGUF Source: Original Platform --- .gitattributes | 62 +++++++++ README.md | 128 ++++++++++++++++++ SmallThinker-21B-A3B-Instruct-QAT.Q4_0.gguf | 3 + SmallThinker-21B-A3B-Instruct-QAT.Q4_K_M.gguf | 3 + SmallThinker-21B-A3B-Instruct-QAT.Q4_K_S.gguf | 3 + SmallThinker-21B-A3B-Instruct.F16.gguf | 3 + SmallThinker-21B-A3B-Instruct.IQ4_NL.gguf | 3 + SmallThinker-21B-A3B-Instruct.IQ4_XS.gguf | 3 + ...inker-21B-A3B-Instruct.IQ4_XS.imatrix.gguf | 3 + SmallThinker-21B-A3B-Instruct.Q3_K.gguf | 3 + ...Thinker-21B-A3B-Instruct.Q3_K.imatrix.gguf | 3 + SmallThinker-21B-A3B-Instruct.Q3_K_S.gguf | 3 + ...inker-21B-A3B-Instruct.Q3_K_S.imatrix.gguf | 3 + SmallThinker-21B-A3B-Instruct.Q4_0.gguf | 3 + ...Thinker-21B-A3B-Instruct.Q4_0.imatrix.gguf | 3 + ...nker-21B-A3B-Instruct.Q4_0.powerinfer.gguf | 3 + SmallThinker-21B-A3B-Instruct.Q4_1.gguf | 3 + ...Thinker-21B-A3B-Instruct.Q4_1.imatrix.gguf | 3 + SmallThinker-21B-A3B-Instruct.Q4_K.gguf | 3 + ...Thinker-21B-A3B-Instruct.Q4_K.imatrix.gguf | 3 + SmallThinker-21B-A3B-Instruct.Q4_K_S.gguf | 3 + ...inker-21B-A3B-Instruct.Q4_K_S.imatrix.gguf | 3 + SmallThinker-21B-A3B-Instruct.Q5_K.gguf | 3 + ...Thinker-21B-A3B-Instruct.Q5_K.imatrix.gguf | 3 + SmallThinker-21B-A3B-Instruct.Q6_K.gguf | 3 + ...Thinker-21B-A3B-Instruct.Q6_K.imatrix.gguf | 3 + SmallThinker-21B-A3B-Instruct.Q8_0.gguf | 3 + ...Thinker-21B-A3B-Instruct.Q8_0.imatrix.gguf | 3 + zh-21b-imatrix.dat | 3 + 29 files changed, 271 insertions(+) create mode 100644 .gitattributes create mode 100644 README.md create mode 100644 SmallThinker-21B-A3B-Instruct-QAT.Q4_0.gguf create mode 100644 SmallThinker-21B-A3B-Instruct-QAT.Q4_K_M.gguf create mode 100644 SmallThinker-21B-A3B-Instruct-QAT.Q4_K_S.gguf create mode 100644 SmallThinker-21B-A3B-Instruct.F16.gguf create mode 100644 SmallThinker-21B-A3B-Instruct.IQ4_NL.gguf create mode 100644 SmallThinker-21B-A3B-Instruct.IQ4_XS.gguf create mode 100644 SmallThinker-21B-A3B-Instruct.IQ4_XS.imatrix.gguf create mode 100644 SmallThinker-21B-A3B-Instruct.Q3_K.gguf create mode 100644 SmallThinker-21B-A3B-Instruct.Q3_K.imatrix.gguf create mode 100644 SmallThinker-21B-A3B-Instruct.Q3_K_S.gguf create mode 100644 SmallThinker-21B-A3B-Instruct.Q3_K_S.imatrix.gguf create mode 100644 SmallThinker-21B-A3B-Instruct.Q4_0.gguf create mode 100644 SmallThinker-21B-A3B-Instruct.Q4_0.imatrix.gguf create mode 100644 SmallThinker-21B-A3B-Instruct.Q4_0.powerinfer.gguf create mode 100644 SmallThinker-21B-A3B-Instruct.Q4_1.gguf create mode 100644 SmallThinker-21B-A3B-Instruct.Q4_1.imatrix.gguf create mode 100644 SmallThinker-21B-A3B-Instruct.Q4_K.gguf create mode 100644 SmallThinker-21B-A3B-Instruct.Q4_K.imatrix.gguf create mode 100644 SmallThinker-21B-A3B-Instruct.Q4_K_S.gguf create mode 100644 SmallThinker-21B-A3B-Instruct.Q4_K_S.imatrix.gguf create mode 100644 SmallThinker-21B-A3B-Instruct.Q5_K.gguf create mode 100644 SmallThinker-21B-A3B-Instruct.Q5_K.imatrix.gguf create mode 100644 SmallThinker-21B-A3B-Instruct.Q6_K.gguf create mode 100644 SmallThinker-21B-A3B-Instruct.Q6_K.imatrix.gguf create mode 100644 SmallThinker-21B-A3B-Instruct.Q8_0.gguf create mode 100644 SmallThinker-21B-A3B-Instruct.Q8_0.imatrix.gguf create mode 100644 zh-21b-imatrix.dat diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..f5e833a --- /dev/null +++ b/.gitattributes @@ -0,0 +1,62 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +SmallThinker-21B-A3B-Instruct.F16.gguf filter=lfs diff=lfs merge=lfs -text +SmallThinker-21B-A3B-Instruct.Q4_0.gguf filter=lfs diff=lfs merge=lfs -text +SmallThinker-21B-A3B-Instruct.Q8_0.gguf filter=lfs diff=lfs merge=lfs -text +SmallThinker-21B-A3B-Instruct.Q4_K.gguf filter=lfs diff=lfs merge=lfs -text +SmallThinker-21B-A3B-Instruct.Q6_K.gguf filter=lfs diff=lfs merge=lfs -text +SmallThinker-21B-A3B-Instruct.Q4_1.gguf filter=lfs diff=lfs merge=lfs -text +SmallThinker-21B-A3B-Instruct.Q3_K.gguf filter=lfs diff=lfs merge=lfs -text +SmallThinker-21B-A3B-Instruct.Q4_K_S.gguf filter=lfs diff=lfs merge=lfs -text +SmallThinker-21B-A3B-Instruct.Q5_K.gguf filter=lfs diff=lfs merge=lfs -text +SmallThinker-21B-A3B-Instruct.Q4_0.powerinfer.gguf filter=lfs diff=lfs merge=lfs -text +SmallThinker-21B-A3B-Instruct.IQ4_NL.gguf filter=lfs diff=lfs merge=lfs -text +SmallThinker-21B-A3B-Instruct.IQ4_XS.gguf filter=lfs diff=lfs merge=lfs -text +SmallThinker-21B-A3B-Instruct.IQ4_XS.imatrix.gguf filter=lfs diff=lfs merge=lfs -text +SmallThinker-21B-A3B-Instruct.Q3_K.imatrix.gguf filter=lfs diff=lfs merge=lfs -text +SmallThinker-21B-A3B-Instruct.Q3_K_S.gguf filter=lfs diff=lfs merge=lfs -text +SmallThinker-21B-A3B-Instruct.Q3_K_S.imatrix.gguf filter=lfs diff=lfs merge=lfs -text +SmallThinker-21B-A3B-Instruct.Q4_0.imatrix.gguf filter=lfs diff=lfs merge=lfs -text +SmallThinker-21B-A3B-Instruct.Q4_1.imatrix.gguf filter=lfs diff=lfs merge=lfs -text +SmallThinker-21B-A3B-Instruct.Q4_K.imatrix.gguf filter=lfs diff=lfs merge=lfs -text +SmallThinker-21B-A3B-Instruct.Q4_K_S.imatrix.gguf filter=lfs diff=lfs merge=lfs -text +SmallThinker-21B-A3B-Instruct.Q5_K.imatrix.gguf filter=lfs diff=lfs merge=lfs -text +SmallThinker-21B-A3B-Instruct.Q6_K.imatrix.gguf filter=lfs diff=lfs merge=lfs -text +SmallThinker-21B-A3B-Instruct.Q8_0.imatrix.gguf filter=lfs diff=lfs merge=lfs -text +zh-21b-imatrix.dat filter=lfs diff=lfs merge=lfs -text +SmallThinker-21B-A3B-Instruct-QAT.Q4_0.gguf filter=lfs diff=lfs merge=lfs -text +SmallThinker-21B-A3B-Instruct-QAT.Q4_K_M.gguf filter=lfs diff=lfs merge=lfs -text +SmallThinker-21B-A3B-Instruct-QAT.Q4_K_S.gguf filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..8f09f0d --- /dev/null +++ b/README.md @@ -0,0 +1,128 @@ +--- +license: apache-2.0 +language: +- en +pipeline_tag: text-generation +base_model: +- PowerInfer/SmallThinker-21BA3B-Instruct +--- +## SmallThinker-21BA3B-Instruct-GGUF + +- GGUF models with `.gguf` suffix can used with [*llama.cpp*](https://github.com/ggml-org/llama.cpp) framework. + +- GGUF models with `.powerinfer.gguf` suffix are integrated with fused sparse FFN operators and sparse LM head operators. These models are only compatible to [*powerinfer*](https://github.com/SJTU-IPADS/PowerInfer/tree/main/smallthinker) framework. + +## Introduction + +

+   ðŸ¤— Hugging Face   |   ðŸ¤– ModelScope   |    📑 Technical Report    +

+ +SmallThinker is a family of **on-device native** Mixture-of-Experts (MoE) language models specially designed for local deployment, +co-developed by the **IPADS** and **School of AI at Shanghai Jiao Tong University** and **Zenergize AI**. +Designed from the ground up for resource-constrained environments, +SmallThinker brings powerful, private, and low-latency AI directly to your personal devices, +without relying on the cloud. + + + +## Performance + +Note: The model is trained mainly on English. + +| Model | MMLU | GPQA-diamond | MATH-500 | IFEVAL | LIVEBENCH | HUMANEVAL | Average | +|------------------------------|-------|--------------|----------|--------|-----------|-----------|---------| +| **SmallThinker-21BA3B-Instruct** | 84.43 | 55.05 | 82.4 | **85.77** | **60.3** | 89.63 | **76.26** | +| Gemma3-12b-it | 78.52 | 34.85 | 82.4 | 74.68 | 44.5 | 82.93 | 66.31 | +| Qwen3-14B | 84.82 | 50 | **84.6** | 85.21| 59.5 | 88.41 | 75.42 | +| Qwen3-30BA3B | **85.1** | 44.4 | 84.4 | 84.29 | 58.8 | **90.24** | 74.54 | +| Qwen3-8B | 81.79 | 38.89 | 81.6 | 83.92 | 49.5 | 85.9 | 70.26 | +| Phi-4-14B | 84.58 | **55.45** | 80.2 | 63.22 | 42.4 | 87.2 | 68.84 | + +For the MMLU evaluation, we use a 0-shot CoT setting. + +All models are evaluated in non-thinking mode. + +## Speed +| Model | Memory(GiB) | i9 14900 | 1+13 8ge4 | rk3588 (16G) | Raspberry PI 5 | +|--------------------------------------|---------------------|----------|-----------|--------------|----------------| +| SmallThinker 21B+sparse | 11.47 | 30.19 | 23.03 | 10.84 | 6.61 | +| SmallThinker 21B+sparse+limited memory | limit 8G | 20.30 | 15.50 | 8.56 | - | +| Qwen3 30B A3B | 16.20 | 33.52 | 20.18 | 9.07 | - | +| Qwen3 30B A3B+limited memory | limit 8G | 10.11 | 0.18 | 6.32 | - | +| Gemma 3n E2B | 1G, theoretically | 36.88 | 27.06 | 12.50 | 6.66 | +| Gemma 3n E4B | 2G, theoretically | 21.93 | 16.58 | 7.37 | 4.01 | + +Note: i9 14900, 1+13 8ge4 use 4 threads, others use the number of threads that can achieve the maximum speed. All models here have been quantized to q4_0. +You can deploy SmallThinker with offloading support using [PowerInfer](https://github.com/SJTU-IPADS/PowerInfer/tree/main/smallthinker) + +## Model Card + +
+ +| **Architecture** | Mixture-of-Experts (MoE) | +|:---:|:---:| +| **Total Parameters** | 21B | +| **Activated Parameters** | 3B | +| **Number of Layers** | 52 | +| **Attention Hidden Dimension** | 2560 | +| **MoE Hidden Dimension** (per Expert) | 768 | +| **Number of Attention Heads** | 28 | +| **Number of KV Heads** | 4 | +| **Number of Experts** | 64 | +| **Selected Experts per Token** | 6 | +| **Vocabulary Size** | 151,936 | +| **Context Length** | 16K | +| **Attention Mechanism** | GQA | +| **Activation Function** | ReGLU | +
+ +## How to Run + +### Transformers + +`transformers==4.53.3` is required, we are actively working to support the latest version. +The following contains a code snippet illustrating how to use the model generate content based on given inputs. + +```python +from transformers import AutoModelForCausalLM, AutoTokenizer +import torch + +path = "PowerInfer/SmallThinker-21BA3B-Instruct" +device = "cuda" + +tokenizer = AutoTokenizer.from_pretrained(path, trust_remote_code=True) +model = AutoModelForCausalLM.from_pretrained(path, torch_dtype=torch.bfloat16, device_map=device, trust_remote_code=True) + +messages = [ + {"role": "user", "content": "Give me a short introduction to large language model."}, +] +model_inputs = tokenizer.apply_chat_template(messages, return_tensors="pt", add_generation_prompt=True).to(device) + +model_outputs = model.generate( + model_inputs, + do_sample=True, + max_new_tokens=1024 +) + +output_token_ids = [ + model_outputs[i][len(model_inputs[i]):] for i in range(len(model_inputs)) +] + +responses = tokenizer.batch_decode(output_token_ids, skip_special_tokens=True)[0] +print(responses) + +``` + +### ModelScope + +`ModelScope` adopts Python API similar to (though not entirely identical to) `Transformers`. For basic usage, simply modify the first line of the above code as follows: + +```python +from modelscope import AutoModelForCausalLM, AutoTokenizer +``` + +## Statement +- Due to the constraints of its model size and the limitations of its training data, its responses may contain factual inaccuracies, biases, or outdated information. +- Users bear full responsibility for independently evaluating and verifying the accuracy and appropriateness of all generated content. +- SmallThinker does not possess genuine comprehension or consciousness and cannot express personal opinions or value judgments. \ No newline at end of file diff --git a/SmallThinker-21B-A3B-Instruct-QAT.Q4_0.gguf b/SmallThinker-21B-A3B-Instruct-QAT.Q4_0.gguf new file mode 100644 index 0000000..819c72b --- /dev/null +++ b/SmallThinker-21B-A3B-Instruct-QAT.Q4_0.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dbc2f9114330a0cae24c7955591e9ed89881ab3c2b292477037b9f967b1b6b33 +size 12233894624 diff --git a/SmallThinker-21B-A3B-Instruct-QAT.Q4_K_M.gguf b/SmallThinker-21B-A3B-Instruct-QAT.Q4_K_M.gguf new file mode 100644 index 0000000..035eb2a --- /dev/null +++ b/SmallThinker-21B-A3B-Instruct-QAT.Q4_K_M.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e0fd3e26e144a8ba22a6de11337bcff40fb4f612ed239710f75444781f7fa0aa +size 13086128864 diff --git a/SmallThinker-21B-A3B-Instruct-QAT.Q4_K_S.gguf b/SmallThinker-21B-A3B-Instruct-QAT.Q4_K_S.gguf new file mode 100644 index 0000000..767e8b4 --- /dev/null +++ b/SmallThinker-21B-A3B-Instruct-QAT.Q4_K_S.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d426d45249b49d409d29e698de09d6bd6756489312fd03bbb55cedd99100e320 +size 12328921824 diff --git a/SmallThinker-21B-A3B-Instruct.F16.gguf b/SmallThinker-21B-A3B-Instruct.F16.gguf new file mode 100644 index 0000000..4a19ba8 --- /dev/null +++ b/SmallThinker-21B-A3B-Instruct.F16.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a05788324cc13299a0e82b016e3b7b9a56d7731badddf8e4d63ed05e62e1c147 +size 43036667072 diff --git a/SmallThinker-21B-A3B-Instruct.IQ4_NL.gguf b/SmallThinker-21B-A3B-Instruct.IQ4_NL.gguf new file mode 100644 index 0000000..6a42e65 --- /dev/null +++ b/SmallThinker-21B-A3B-Instruct.IQ4_NL.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:22cffa938deaa492bd3b045ab78490e1a06a9782b93ad44e8f4ccd35e7a3231f +size 12336786112 diff --git a/SmallThinker-21B-A3B-Instruct.IQ4_XS.gguf b/SmallThinker-21B-A3B-Instruct.IQ4_XS.gguf new file mode 100644 index 0000000..7d524c7 --- /dev/null +++ b/SmallThinker-21B-A3B-Instruct.IQ4_XS.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b0d0f2c322b9c57b56043eb5024918f8faf4f52cfd052f12f1d456455a259755 +size 11702858432 diff --git a/SmallThinker-21B-A3B-Instruct.IQ4_XS.imatrix.gguf b/SmallThinker-21B-A3B-Instruct.IQ4_XS.imatrix.gguf new file mode 100644 index 0000000..487a7e7 --- /dev/null +++ b/SmallThinker-21B-A3B-Instruct.IQ4_XS.imatrix.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b3308a7b96b10c2c80dd08355f59f481d7ff0a2bd6409d3742247e0734468fe6 +size 11584893920 diff --git a/SmallThinker-21B-A3B-Instruct.Q3_K.gguf b/SmallThinker-21B-A3B-Instruct.Q3_K.gguf new file mode 100644 index 0000000..8d1fa00 --- /dev/null +++ b/SmallThinker-21B-A3B-Instruct.Q3_K.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a28f660926230d924ff0dabbcbdcc05c2ed0e66de788dc25d3285f21e5ca4a6e +size 10419317952 diff --git a/SmallThinker-21B-A3B-Instruct.Q3_K.imatrix.gguf b/SmallThinker-21B-A3B-Instruct.Q3_K.imatrix.gguf new file mode 100644 index 0000000..2a2f790 --- /dev/null +++ b/SmallThinker-21B-A3B-Instruct.Q3_K.imatrix.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ab85a8c438260a0892ef5729ef5ad86fc086bc2b1aba19d4c8671b34d3f6c66b +size 10419318240 diff --git a/SmallThinker-21B-A3B-Instruct.Q3_K_S.gguf b/SmallThinker-21B-A3B-Instruct.Q3_K_S.gguf new file mode 100644 index 0000000..22f5e2a --- /dev/null +++ b/SmallThinker-21B-A3B-Instruct.Q3_K_S.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e1ee54d20ac9cc0f27e5342c7300f9e0a74629910c33708a8b693ba33faaf1c3 +size 9430379712 diff --git a/SmallThinker-21B-A3B-Instruct.Q3_K_S.imatrix.gguf b/SmallThinker-21B-A3B-Instruct.Q3_K_S.imatrix.gguf new file mode 100644 index 0000000..3015b68 --- /dev/null +++ b/SmallThinker-21B-A3B-Instruct.Q3_K_S.imatrix.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:17107f61802655eec0d9c61f952c74806a61eaa71818012484f13ba80aa7d3ca +size 9430380000 diff --git a/SmallThinker-21B-A3B-Instruct.Q4_0.gguf b/SmallThinker-21B-A3B-Instruct.Q4_0.gguf new file mode 100644 index 0000000..2d64e88 --- /dev/null +++ b/SmallThinker-21B-A3B-Instruct.Q4_0.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a4c742630d85eb8bf81b6152cb58fd01aeef26eac68a203ac34121592fc69718 +size 12233894592 diff --git a/SmallThinker-21B-A3B-Instruct.Q4_0.imatrix.gguf b/SmallThinker-21B-A3B-Instruct.Q4_0.imatrix.gguf new file mode 100644 index 0000000..c6f625d --- /dev/null +++ b/SmallThinker-21B-A3B-Instruct.Q4_0.imatrix.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0d72e21c5859927a1b8737047215f20a2a95b0a1321491ca34c43d131dc7ea26 +size 12281080800 diff --git a/SmallThinker-21B-A3B-Instruct.Q4_0.powerinfer.gguf b/SmallThinker-21B-A3B-Instruct.Q4_0.powerinfer.gguf new file mode 100644 index 0000000..342f804 --- /dev/null +++ b/SmallThinker-21B-A3B-Instruct.Q4_0.powerinfer.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9bb2ad80adf5f64cb953570690cf1aee13de87661eef65f073224c9a3afe4ccb +size 12144740608 diff --git a/SmallThinker-21B-A3B-Instruct.Q4_1.gguf b/SmallThinker-21B-A3B-Instruct.Q4_1.gguf new file mode 100644 index 0000000..866784e --- /dev/null +++ b/SmallThinker-21B-A3B-Instruct.Q4_1.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:390006ffe10c9c7cba56223e8c83a1d107cf14dae2933b6621a2e710b53b44cc +size 13553195712 diff --git a/SmallThinker-21B-A3B-Instruct.Q4_1.imatrix.gguf b/SmallThinker-21B-A3B-Instruct.Q4_1.imatrix.gguf new file mode 100644 index 0000000..ca3ff3d --- /dev/null +++ b/SmallThinker-21B-A3B-Instruct.Q4_1.imatrix.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b7ad6b48093fb1fff7d9ee1bddfa6d9f00403ad16573eeabd031f4fb245585ee +size 13553196000 diff --git a/SmallThinker-21B-A3B-Instruct.Q4_K.gguf b/SmallThinker-21B-A3B-Instruct.Q4_K.gguf new file mode 100644 index 0000000..6f4b868 --- /dev/null +++ b/SmallThinker-21B-A3B-Instruct.Q4_K.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5986bc7f0539bd87da68ae9bc7c2156a545b28bdf46dc9d896030d1d1ebf773 +size 13086128832 diff --git a/SmallThinker-21B-A3B-Instruct.Q4_K.imatrix.gguf b/SmallThinker-21B-A3B-Instruct.Q4_K.imatrix.gguf new file mode 100644 index 0000000..7dfba94 --- /dev/null +++ b/SmallThinker-21B-A3B-Instruct.Q4_K.imatrix.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:17724518e0dbf3c8f384db973194afe0e5dd3c41dd6d5cc2799ba7a102257395 +size 13086129120 diff --git a/SmallThinker-21B-A3B-Instruct.Q4_K_S.gguf b/SmallThinker-21B-A3B-Instruct.Q4_K_S.gguf new file mode 100644 index 0000000..de46ed7 --- /dev/null +++ b/SmallThinker-21B-A3B-Instruct.Q4_K_S.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ad109d1b643c5b2df3916686ef43006905c8cc726d9b47f6ca916aaa51bf5165 +size 12328921792 diff --git a/SmallThinker-21B-A3B-Instruct.Q4_K_S.imatrix.gguf b/SmallThinker-21B-A3B-Instruct.Q4_K_S.imatrix.gguf new file mode 100644 index 0000000..dd07a54 --- /dev/null +++ b/SmallThinker-21B-A3B-Instruct.Q4_K_S.imatrix.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:24a755b3e1be10e7de82dcb43ba896ab13271241cc862af626d1a4c0dd2c3b22 +size 12328922080 diff --git a/SmallThinker-21B-A3B-Instruct.Q5_K.gguf b/SmallThinker-21B-A3B-Instruct.Q5_K.gguf new file mode 100644 index 0000000..cf1a604 --- /dev/null +++ b/SmallThinker-21B-A3B-Instruct.Q5_K.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c7ee045467936dc3f79d51a46e9149c18c6d2f074e6979fce61763bb6eecbfbc +size 15311526592 diff --git a/SmallThinker-21B-A3B-Instruct.Q5_K.imatrix.gguf b/SmallThinker-21B-A3B-Instruct.Q5_K.imatrix.gguf new file mode 100644 index 0000000..b55ed35 --- /dev/null +++ b/SmallThinker-21B-A3B-Instruct.Q5_K.imatrix.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5297f8992520bd44638cae2cb6ee9e1d729b1afb38b659505985abc8de679b9 +size 15311526880 diff --git a/SmallThinker-21B-A3B-Instruct.Q6_K.gguf b/SmallThinker-21B-A3B-Instruct.Q6_K.gguf new file mode 100644 index 0000000..38100dd --- /dev/null +++ b/SmallThinker-21B-A3B-Instruct.Q6_K.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1980abe422094fde43fae2a969415f7e1e1edc15ea97787be192646637cdc56c +size 17676011712 diff --git a/SmallThinker-21B-A3B-Instruct.Q6_K.imatrix.gguf b/SmallThinker-21B-A3B-Instruct.Q6_K.imatrix.gguf new file mode 100644 index 0000000..2507c04 --- /dev/null +++ b/SmallThinker-21B-A3B-Instruct.Q6_K.imatrix.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2cd33bfa8395efee5a88f2d526db44d623a40ff93130dcb91c66c1d1571a26f4 +size 17676012000 diff --git a/SmallThinker-21B-A3B-Instruct.Q8_0.gguf b/SmallThinker-21B-A3B-Instruct.Q8_0.gguf new file mode 100644 index 0000000..12891d6 --- /dev/null +++ b/SmallThinker-21B-A3B-Instruct.Q8_0.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:28658c177ac6ce90f4ca95deda0e6199623acdfa0f7a78ba6f135afda7af2b96 +size 22882503872 diff --git a/SmallThinker-21B-A3B-Instruct.Q8_0.imatrix.gguf b/SmallThinker-21B-A3B-Instruct.Q8_0.imatrix.gguf new file mode 100644 index 0000000..8e948fa --- /dev/null +++ b/SmallThinker-21B-A3B-Instruct.Q8_0.imatrix.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9b61f284db0b59a334f3bf2a56cf961e5000f066e1a8a82ef97016424efd9f62 +size 22882504160 diff --git a/zh-21b-imatrix.dat b/zh-21b-imatrix.dat new file mode 100644 index 0000000..c72ab14 --- /dev/null +++ b/zh-21b-imatrix.dat @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:92dbbe39026cb46bee51e93063a71667f78cc5211534043191b5f27a01256dda +size 81271293