From 5181b294bd7472a41af3bc0a17a4e8c5aac56c65 Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Fri, 10 Jul 2026 08:22:12 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: Tiiny/SmallThinker-4BA0.6B-Instruct-GGUF Source: Original Platform --- .gitattributes | 49 +++++++ README.md | 127 ++++++++++++++++++ SmallThinker-4B-A0.6B-Instruct.F16.gguf | 3 + SmallThinker-4B-A0.6B-Instruct.IQ4_NL.gguf | 3 + SmallThinker-4B-A0.6B-Instruct.IQ4_XS.gguf | 3 + SmallThinker-4B-A0.6B-Instruct.Q3_K.gguf | 3 + SmallThinker-4B-A0.6B-Instruct.Q3_K_S.gguf | 3 + SmallThinker-4B-A0.6B-Instruct.Q4_0.gguf | 3 + ...ker-4B-A0.6B-Instruct.Q4_0.powerinfer.gguf | 3 + SmallThinker-4B-A0.6B-Instruct.Q4_1.gguf | 3 + SmallThinker-4B-A0.6B-Instruct.Q4_K.gguf | 3 + SmallThinker-4B-A0.6B-Instruct.Q4_K_S.gguf | 3 + SmallThinker-4B-A0.6B-Instruct.Q5_K.gguf | 3 + SmallThinker-4B-A0.6B-Instruct.Q6_K.gguf | 3 + SmallThinker-4B-A0.6B-Instruct.Q8_0.gguf | 3 + 15 files changed, 215 insertions(+) create mode 100644 .gitattributes create mode 100644 README.md create mode 100644 SmallThinker-4B-A0.6B-Instruct.F16.gguf create mode 100644 SmallThinker-4B-A0.6B-Instruct.IQ4_NL.gguf create mode 100644 SmallThinker-4B-A0.6B-Instruct.IQ4_XS.gguf create mode 100644 SmallThinker-4B-A0.6B-Instruct.Q3_K.gguf create mode 100644 SmallThinker-4B-A0.6B-Instruct.Q3_K_S.gguf create mode 100644 SmallThinker-4B-A0.6B-Instruct.Q4_0.gguf create mode 100644 SmallThinker-4B-A0.6B-Instruct.Q4_0.powerinfer.gguf create mode 100644 SmallThinker-4B-A0.6B-Instruct.Q4_1.gguf create mode 100644 SmallThinker-4B-A0.6B-Instruct.Q4_K.gguf create mode 100644 SmallThinker-4B-A0.6B-Instruct.Q4_K_S.gguf create mode 100644 SmallThinker-4B-A0.6B-Instruct.Q5_K.gguf create mode 100644 SmallThinker-4B-A0.6B-Instruct.Q6_K.gguf create mode 100644 SmallThinker-4B-A0.6B-Instruct.Q8_0.gguf diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..811c71c --- /dev/null +++ b/.gitattributes @@ -0,0 +1,49 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +SmallThinker-4B-A0.6B-Instruct.F16.gguf filter=lfs diff=lfs merge=lfs -text +SmallThinker-4B-A0.6B-Instruct.Q4_0.gguf filter=lfs diff=lfs merge=lfs -text +SmallThinker-4B-A0.6B-Instruct.Q8_0.gguf filter=lfs diff=lfs merge=lfs -text +SmallThinker-4B-A0.6B-Instruct.Q4_K.gguf filter=lfs diff=lfs merge=lfs -text +SmallThinker-4B-A0.6B-Instruct.Q6_K.gguf filter=lfs diff=lfs merge=lfs -text +SmallThinker-4B-A0.6B-Instruct.Q4_1.gguf filter=lfs diff=lfs merge=lfs -text +SmallThinker-21B-A3B-Instruct.Q4_1.gguf filter=lfs diff=lfs merge=lfs -text +SmallThinker-4B-A0.6B-Instruct.Q3_K.gguf filter=lfs diff=lfs merge=lfs -text +SmallThinker-4B-A0.6B-Instruct.Q4_K_S.gguf filter=lfs diff=lfs merge=lfs -text +SmallThinker-4B-A0.6B-Instruct.Q5_K.gguf filter=lfs diff=lfs merge=lfs -text +SmallThinker-4B-A0.6B-Instruct.Q4_0.powerinfer.gguf filter=lfs diff=lfs merge=lfs -text +SmallThinker-4B-A0.6B-Instruct.IQ4_NL.gguf filter=lfs diff=lfs merge=lfs -text +SmallThinker-4B-A0.6B-Instruct.IQ4_XS.gguf filter=lfs diff=lfs merge=lfs -text +SmallThinker-4B-A0.6B-Instruct.Q3_K_S.gguf filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..264bab9 --- /dev/null +++ b/README.md @@ -0,0 +1,127 @@ +--- +license: apache-2.0 +language: +- en +pipeline_tag: text-generation +base_model: +- PowerInfer/SmallThinker-4BA0.6B-Instruct +--- +## SmallThinker-4BA0.6B-Instruct-GGUF + +- GGUF models with `.gguf` suffix can used with [*llama.cpp*](https://github.com/ggml-org/llama.cpp) framework. + +- GGUF models with `.powerinfer.gguf` suffix are integrated with fused sparse FFN operators and sparse LM head operators. These models are only compatible to [*powerinfer*](https://github.com/SJTU-IPADS/PowerInfer/tree/main/smallthinker) framework. + + +## Introduction + +

+   ðŸ¤— Hugging Face   |   ðŸ¤– ModelScope   |    📑 Technical Report    +

+ +SmallThinker is a family of **on-device native** Mixture-of-Experts (MoE) language models specially designed for local deployment, +co-developed by the **IPADS and School of AI at Shanghai Jiao Tong University** and **Zenergize AI**. +Designed from the ground up for resource-constrained environments, +SmallThinker brings powerful, private, and low-latency AI directly to your personal devices, +without relying on the cloud. + +## Performance + +Note: The model is trained mainly on English. + +| Model | MMLU | GPQA-diamond | GSM8K | MATH-500 | IFEVAL | LIVEBENCH | HUMANEVAL | Average | +| :--- | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | +| **SmallThinker-4BA0.6B-Instruct** | **66.11** | **31.31** | 80.02 | 60.60 | 69.69 | **42.20** | **82.32** | **61.75** | +| Qwen3-0.6B | 43.31 | 26.77 | 62.85 | 45.6 | 58.41 | 23.1 | 31.71 | 41.67 | +| Qwen3-1.7B | 64.19 | 27.78 | 81.88 | **63.6** | 69.50 | 35.60 | 61.59 | 57.73 | +| Gemma3nE2b-it | 63.04 | 20.2 | **82.34** | 58.6 | **73.2** | 27.90 | 64.63 | 55.70 | +| Llama-3.2-3B-Instruct | 64.15 | 24.24 | 75.51 | 40 | 71.16 | 15.30 | 55.49 | 49.41 | +| Llama-3.2-1B-Instruct | 45.66 | 22.73 | 1.67 | 14.4 | 48.06 | 13.50 | 37.20 | 26.17 | + +For the MMLU evaluation, we use a 0-shot CoT setting. + +All models are evaluated in non-thinking mode. + + +## Speed +| Model | Memory(GiB) | i9 14900 | 1+13 8gen4 | rk3588 (16G) | rk3576 | Raspberry PI 5 | RDK X5 | rk3566 | +|-----------------------------------------------|---------------------|----------|------------|--------------|--------|----------------|--------|--------| +| SmallThinker 4B+sparse ffn +sparse lm_head | 2.24 | 108.17 | 78.99 | 39.76 | 15.10 | 28.77 | 7.23 | 6.33 | +| SmallThinker 4B+sparse ffn +sparse lm_head+limited memory | limit 1G| 29.99 | 20.91 | 15.04 | 2.60 | 0.75 | 0.67 | 0.74 | +| Qwen3 0.6B | 0.6 | 148.56 | 94.91 | 45.93 | 15.29 | 27.44 | 13.32 | 9.76 | +| Qwen3 1.7B | 1.3 | 62.24 | 41.00 | 20.29 | 6.09 | 11.08 | 6.35 | 4.15 | +| Qwen3 1.7B+limited memory | limit 1G | 2.66 | 1.09 | 1.00 | 0.47 | - | - | 0.11 | +| Gemma3n E2B | 1G, theoretically | 36.88 | 27.06 | 12.50 | 3.80 | 6.66 | 3.46 | 2.45 | + +Note: i9 14900, 1+13 8ge4 use 4 threads, others use the number of threads that can achieve the maximum speed. All models here have been quantized to q4_0. + +You can deploy SmallThinker with offloading support using [PowerInfer](https://github.com/SJTU-IPADS/PowerInfer/tree/main/smallthinker) + +## Model Card + +
+ +| **Architecture** | Mixture-of-Experts (MoE) | +|:---:|:---:| +| **Total Parameters** | 4B | +| **Activated Parameters** | 0.6B | +| **Number of Layers** | 32 | +| **Attention Hidden Dimension** | 1536 | +| **MoE Hidden Dimension** (per Expert) | 768 | +| **Number of Attention Heads** | 12 | +| **Number of Experts** | 32 | +| **Selected Experts per Token** | 4 | +| **Vocabulary Size** | 151,936 | +| **Context Length** | 32K | +| **Attention Mechanism** | GQA | +| **Activation Function** | ReGLU | +
+ +## How to Run + +### Transformers + +`transformers==4.53.3` is required, we are actively working to support the latest version. +The following contains a code snippet illustrating how to use the model generate content based on given inputs. + +```python +from transformers import AutoModelForCausalLM, AutoTokenizer +import torch + +path = "PowerInfer/SmallThinker-4BA0.6B-Instruct" +device = "cuda" + +tokenizer = AutoTokenizer.from_pretrained(path, trust_remote_code=True) +model = AutoModelForCausalLM.from_pretrained(path, torch_dtype=torch.bfloat16, device_map=device, trust_remote_code=True) + +messages = [ + {"role": "user", "content": "Give me a short introduction to large language model."}, +] +model_inputs = tokenizer.apply_chat_template(messages, return_tensors="pt", add_generation_prompt=True).to(device) + +model_outputs = model.generate( + model_inputs, + do_sample=True, + max_new_tokens=1024 +) + +output_token_ids = [ + model_outputs[i][len(model_inputs[i]):] for i in range(len(model_inputs)) +] + +responses = tokenizer.batch_decode(output_token_ids, skip_special_tokens=True)[0] +print(responses) + +``` + +### ModelScope + +`ModelScope` adopts Python API similar to (though not entirely identical to) `Transformers`. For basic usage, simply modify the first line of the above code as follows: + +```python +from modelscope import AutoModelForCausalLM, AutoTokenizer +``` +## Statement +- Due to the constraints of its model size and the limitations of its training data, its responses may contain factual inaccuracies, biases, or outdated information. +- Users bear full responsibility for independently evaluating and verifying the accuracy and appropriateness of all generated content. +- SmallThinker does not possess genuine comprehension or consciousness and cannot express personal opinions or value judgments. \ No newline at end of file diff --git a/SmallThinker-4B-A0.6B-Instruct.F16.gguf b/SmallThinker-4B-A0.6B-Instruct.F16.gguf new file mode 100644 index 0000000..1bf11cb --- /dev/null +++ b/SmallThinker-4B-A0.6B-Instruct.F16.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:342c47ce4f042bcd8409b7fc0a2f500f40048f7f25578d336ba471e68743ff90 +size 8546216544 diff --git a/SmallThinker-4B-A0.6B-Instruct.IQ4_NL.gguf b/SmallThinker-4B-A0.6B-Instruct.IQ4_NL.gguf new file mode 100644 index 0000000..3cf7868 --- /dev/null +++ b/SmallThinker-4B-A0.6B-Instruct.IQ4_NL.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d391b5c5cfe14e1dce87b698212ec9e83083b6eaccc115dd6483819bd109ce72 +size 2493324384 diff --git a/SmallThinker-4B-A0.6B-Instruct.IQ4_XS.gguf b/SmallThinker-4B-A0.6B-Instruct.IQ4_XS.gguf new file mode 100644 index 0000000..2a4ee7b --- /dev/null +++ b/SmallThinker-4B-A0.6B-Instruct.IQ4_XS.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:999f94f6577087227861e73887458de39a86b386a40a059f13bd8b137b60639d +size 2372392032 diff --git a/SmallThinker-4B-A0.6B-Instruct.Q3_K.gguf b/SmallThinker-4B-A0.6B-Instruct.Q3_K.gguf new file mode 100644 index 0000000..5f9a801 --- /dev/null +++ b/SmallThinker-4B-A0.6B-Instruct.Q3_K.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:93c47555f568f18acb19f873e66a2beddabe82936a4b360b0e9e1c15b55e7155 +size 2118855264 diff --git a/SmallThinker-4B-A0.6B-Instruct.Q3_K_S.gguf b/SmallThinker-4B-A0.6B-Instruct.Q3_K_S.gguf new file mode 100644 index 0000000..8cb69fe --- /dev/null +++ b/SmallThinker-4B-A0.6B-Instruct.Q3_K_S.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:90a2f1dfc55642713ef4ec29b6521c33fd7a0b033522988fe92654805589513a +size 1937189472 diff --git a/SmallThinker-4B-A0.6B-Instruct.Q4_0.gguf b/SmallThinker-4B-A0.6B-Instruct.Q4_0.gguf new file mode 100644 index 0000000..92d9dbf --- /dev/null +++ b/SmallThinker-4B-A0.6B-Instruct.Q4_0.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:47dfade10633c1a10cd51aa8b3bd495cefaeea6396168ddaf35fb4d8e90d7e82 +size 2472877152 diff --git a/SmallThinker-4B-A0.6B-Instruct.Q4_0.powerinfer.gguf b/SmallThinker-4B-A0.6B-Instruct.Q4_0.powerinfer.gguf new file mode 100644 index 0000000..ce6cd44 --- /dev/null +++ b/SmallThinker-4B-A0.6B-Instruct.Q4_0.powerinfer.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:defec76c786476ac0e00baed53e181ec595a203bed9d8318611a291245b5149c +size 2292487712 diff --git a/SmallThinker-4B-A0.6B-Instruct.Q4_1.gguf b/SmallThinker-4B-A0.6B-Instruct.Q4_1.gguf new file mode 100644 index 0000000..dfc77ba --- /dev/null +++ b/SmallThinker-4B-A0.6B-Instruct.Q4_1.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8b7a0e05b88e01e4c343626d4e60b557f7d3ce718c0b45412bfb456be16fc6f2 +size 2724965472 diff --git a/SmallThinker-4B-A0.6B-Instruct.Q4_K.gguf b/SmallThinker-4B-A0.6B-Instruct.Q4_K.gguf new file mode 100644 index 0000000..b378193 --- /dev/null +++ b/SmallThinker-4B-A0.6B-Instruct.Q4_K.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f9e03828eaebdeef503b907f7bc0fabfd4392bdbd8edc9f5433c8530d7e5a6fb +size 2630212704 diff --git a/SmallThinker-4B-A0.6B-Instruct.Q4_K_S.gguf b/SmallThinker-4B-A0.6B-Instruct.Q4_K_S.gguf new file mode 100644 index 0000000..eb29d62 --- /dev/null +++ b/SmallThinker-4B-A0.6B-Instruct.Q4_K_S.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f75612a1ad02248a1f902082030d04cd26f856237cd104acc43a3cf4b2582526 +size 2491948128 diff --git a/SmallThinker-4B-A0.6B-Instruct.Q5_K.gguf b/SmallThinker-4B-A0.6B-Instruct.Q5_K.gguf new file mode 100644 index 0000000..1ea8eb1 --- /dev/null +++ b/SmallThinker-4B-A0.6B-Instruct.Q5_K.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:404a36fe70108d4dc43ce5f8c8de01c49e5b63b46bfc2df3bcd186ee7ec2d025 +size 3058105440 diff --git a/SmallThinker-4B-A0.6B-Instruct.Q6_K.gguf b/SmallThinker-4B-A0.6B-Instruct.Q6_K.gguf new file mode 100644 index 0000000..30ef51f --- /dev/null +++ b/SmallThinker-4B-A0.6B-Instruct.Q6_K.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:539e8d845c6b6cb34c1b27394b2b335ebe9046e8bece50ff35195deae4824f5c +size 3512741472 diff --git a/SmallThinker-4B-A0.6B-Instruct.Q8_0.gguf b/SmallThinker-4B-A0.6B-Instruct.Q8_0.gguf new file mode 100644 index 0000000..8f94fc9 --- /dev/null +++ b/SmallThinker-4B-A0.6B-Instruct.Q8_0.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:27031bfb9162604580afa47c827b42d90fe82f8e39d21a6599d4bad51ca998f3 +size 4546103904