commit 3cff135cdf96d99048eb9ba691e3a15dfe7e7cb9 Author: ModelHub XC Date: Thu Sep 24 14:51:16 2026 +0800 初始化项目，由ModelHub XC社区提供模型 Model: gabriellarson/SmallThinker-4BA0.6B-Instruct-GGUF Source: Original Platform diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..e2164a8 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,54 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +SmallThinker-4BA0.6B-Instruct-Q5_K_M.gguf filter=lfs diff=lfs merge=lfs -text +SmallThinker-4BA0.6B-Instruct-Q6_K.gguf filter=lfs diff=lfs merge=lfs -text +SmallThinker-4BA0.6B-Instruct-Q8_0.gguf filter=lfs diff=lfs merge=lfs -text +SmallThinker-32x758M-4BA0.6B-Instruct-F16.gguf filter=lfs diff=lfs merge=lfs -text +SmallThinker-4BA0.6B-Instruct-Q4_0.gguf filter=lfs diff=lfs merge=lfs -text +SmallThinker-4BA0.6B-Instruct-Q4_K_M.gguf filter=lfs diff=lfs merge=lfs -text +SmallThinker-4BA0.6B-Instruct-Q4_K_S.gguf filter=lfs diff=lfs merge=lfs -text +SmallThinker-4BA0.6B-Instruct-Q5_0.gguf filter=lfs diff=lfs merge=lfs -text +SmallThinker-4BA0.6B-Instruct-Q5_K_S.gguf filter=lfs diff=lfs merge=lfs -text +SmallThinker-4BA0.6B-Instruct-IQ4_XS.gguf filter=lfs diff=lfs merge=lfs -text +SmallThinker-4BA0.6B-Instruct-Q3_K_L.gguf filter=lfs diff=lfs merge=lfs -text +SmallThinker-4BA0.6B-Instruct-Q3_K_M.gguf filter=lfs diff=lfs merge=lfs -text +SmallThinker-4BA0.6B-Instruct-IQ3_XS.gguf filter=lfs diff=lfs merge=lfs -text +SmallThinker-4BA0.6B-Instruct-Q2_K.gguf filter=lfs diff=lfs merge=lfs -text +SmallThinker-4BA0.6B-Instruct-Q2_K_S.gguf filter=lfs diff=lfs merge=lfs -text +SmallThinker-4BA0.6B-Instruct-IQ2_M.gguf filter=lfs diff=lfs merge=lfs -text +SmallThinker-4BA0.6B-Instruct-IQ2_S.gguf filter=lfs diff=lfs merge=lfs -text +SmallThinker-4BA0.6B-Instruct-IQ2_XS.gguf filter=lfs diff=lfs merge=lfs -text +SmallThinker-4BA0.6B-Instruct-IQ2_XXS.gguf filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..b290cb0 --- /dev/null +++ b/README.md @@ -0,0 +1,98 @@ +--- +license: apache-2.0 +language: +- en +pipeline_tag: text-generation +base_model: +- PowerInfer/SmallThinker-4BA0.6B-Instruct +--- +## Introduction + +SmallThinker is a family of **on-device native** Mixture-of-Experts (MoE) language models specially designed for local deployment, +co-developed by the **IPADS and School of AI at Shanghai Jiao Tong University** and **Zenergize AI**. +Designed from the ground up for resource-constrained environments, +SmallThinker brings powerful, private, and low-latency AI directly to your personal devices, +without relying on the cloud. + +## Performance + +Note: The model is trained mainly on English. + +| Model | MMLU | GPQA-diamond | GSM8K | MATH-500 | IFEVAL | LIVEBENCH | HUMANEVAL | Average | +| :--- | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | +| **SmallThinker-4BA0.6B-Instruct** | **66.11** | **31.31** | 80.02 | 60.60 | 69.69 | **42.20** | **82.32** | **61.75** | +| Qwen3-0.6B | 43.31 | 26.77 | 62.85 | 45.6 | 58.41 | 23.1 | 31.71 | 41.67 | +| Qwen3-1.7B | 64.19 | 27.78 | 81.88 | **63.6** | 69.50 | 35.60 | 61.59 | 57.73 | +| Gemma3nE2b-it | 63.04 | 20.2 | **82.34** | 58.6 | **73.2** | 27.90 | 64.63 | 55.70 | +| Llama-3.2-3B-Instruct | 64.15 | 24.24 | 75.51 | 40 | 71.16 | 15.30 | 55.49 | 49.41 | +| Llama-3.2-1B-Instruct | 45.66 | 22.73 | 1.67 | 14.4 | 48.06 | 13.50 | 37.20 | 26.17 | + +For the MMLU evaluation, we use a 0-shot CoT setting. + +All models are evaluated in non-thinking mode. + + +## Speed +| Model | Memory(GiB) | i9 14900 | 1+13 8gen4 | rk3588 (16G) | rk3576 | Raspberry PI 5 | RDK X5 | rk3566 | +|-----------------------------------------------|---------------------|----------|------------|--------------|--------|----------------|--------|--------| +| SmallThinker 4B+sparse ffn +sparse lm_head | 2.24 | 108.17 | 78.99 | 39.76 | 15.10 | 28.77 | 7.23 | 6.33 | +| SmallThinker 4B+sparse ffn +sparse lm_head+limited memory | limit 1G| 29.99 | 20.91 | 15.04 | 2.60 | 0.75 | 0.67 | 0.74 | +| Qwen3 0.6B | 0.6 | 148.56 | 94.91 | 45.93 | 15.29 | 27.44 | 13.32 | 9.76 | +| Qwen3 1.7B | 1.3 | 62.24 | 41.00 | 20.29 | 6.09 | 11.08 | 6.35 | 4.15 | +| Qwen3 1.7B+limited memory | limit 1G | 2.66 | 1.09 | 1.00 | 0.47 | - | - | 0.11 | +| Gemma3n E2B | 1G, theoretically | 36.88 | 27.06 | 12.50 | 3.80 | 6.66 | 3.46 | 2.45 | + +Note: i9 14900, 1+13 8ge4 use 4 threads, others use the number of threads that can achieve the maximum speed. All models here have been quantized to q4_0. +You can deploy SmallThinker with offloading support using [PowerInfer](https://github.com/SJTU-IPADS/PowerInfer/tree/main/smallthinker) +## Model Card +
+| **Architecture** | Mixture-of-Experts (MoE) | +|:---:|:---:| +| **Total Parameters** | 4B | +| **Activated Parameters** | 0.6B | +| **Number of Layers** | 32 | +| **Attention Hidden Dimension** | 1536 | +| **MoE Hidden Dimension** (per Expert) | 768 | +| **Number of Attention Heads** | 12 | +| **Number of Experts** | 32 | +| **Selected Experts per Token** | 4 | +| **Vocabulary Size** | 151,936 | +| **Context Length** | 32K | +| **Attention Mechanism** | GQA | +| **Activation Function** | ReGLU | +
+## How to Run +### Transformers +`transformers==4.53.3` is required, we are actively working to support the latest version. +The following contains a code snippet illustrating how to use the model generate content based on given inputs. +```python +from transformers import AutoModelForCausalLM, AutoTokenizer +import torch +path = "PowerInfer/SmallThinker-4BA0.6B-Instruct" +device = "cuda" +tokenizer = AutoTokenizer.from_pretrained(path, trust_remote_code=True) +model = AutoModelForCausalLM.from_pretrained(path, torch_dtype=torch.bfloat16, device_map=device, trust_remote_code=True) +messages = [ + {"role": "user", "content": "Give me a short introduction to large language model."}, +] +model_inputs = tokenizer.apply_chat_template(messages, return_tensors="pt", add_generation_prompt=True).to(device) +model_outputs = model.generate( + model_inputs, + do_sample=True, + max_new_tokens=1024 +) +output_token_ids = [ + model_outputs[i][len(model_inputs[i]):] for i in range(len(model_inputs)) +] +responses = tokenizer.batch_decode(output_token_ids, skip_special_tokens=True)[0] +print(responses) +``` +### ModelScope +`ModelScope` adopts Python API similar to (though not entirely identical to) `Transformers`. For basic usage, simply modify the first line of the above code as follows: +```python +from modelscope import AutoModelForCausalLM, AutoTokenizer +``` +## Statement +- Due to the constraints of its model size and the limitations of its training data, its responses may contain factual inaccuracies, biases, or outdated information. +- Users bear full responsibility for independently evaluating and verifying the accuracy and appropriateness of all generated content. +- SmallThinker does not possess genuine comprehension or consciousness and cannot express personal opinions or value judgments. \ No newline at end of file diff --git a/SmallThinker-32x758M-4BA0.6B-Instruct-F16.gguf b/SmallThinker-32x758M-4BA0.6B-Instruct-F16.gguf new file mode 100644 index 0000000..bfd6a7c --- /dev/null +++ b/SmallThinker-32x758M-4BA0.6B-Instruct-F16.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:25216dd9158b2498841013a526674b5694221b125a11d2ffee664403fc7dca01 +size 8546216768 diff --git a/SmallThinker-4BA0.6B-Instruct-IQ2_M.gguf b/SmallThinker-4BA0.6B-Instruct-IQ2_M.gguf new file mode 100644 index 0000000..8a4d9ba --- /dev/null +++ b/SmallThinker-4BA0.6B-Instruct-IQ2_M.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cad453bab376c91c6c8b9c86db60ff5a09ce19fc36b5600892368b07a4a2d75b +size 1518385728 diff --git a/SmallThinker-4BA0.6B-Instruct-IQ2_S.gguf b/SmallThinker-4BA0.6B-Instruct-IQ2_S.gguf new file mode 100644 index 0000000..cc5bb2d --- /dev/null +++ b/SmallThinker-4BA0.6B-Instruct-IQ2_S.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1ddc9bc346f9425176e6eb95b3889fb85191ef1bcdd10ac0a6e37aceef73dac1 +size 1407105600 diff --git a/SmallThinker-4BA0.6B-Instruct-IQ2_XS.gguf b/SmallThinker-4BA0.6B-Instruct-IQ2_XS.gguf new file mode 100644 index 0000000..0e1e0ad --- /dev/null +++ b/SmallThinker-4BA0.6B-Instruct-IQ2_XS.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:62eb211ee960baf43cd7a387e7b05c588eab8d7d81eea0e55cf7e1b6ce8699b3 +size 1357451328 diff --git a/SmallThinker-4BA0.6B-Instruct-IQ2_XXS.gguf b/SmallThinker-4BA0.6B-Instruct-IQ2_XXS.gguf new file mode 100644 index 0000000..9b09d75 --- /dev/null +++ b/SmallThinker-4BA0.6B-Instruct-IQ2_XXS.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bc0b24d847c20e7b570f689c01ab622890e6def6345987c35922cac684a88725 +size 1243811904 diff --git a/SmallThinker-4BA0.6B-Instruct-IQ3_XS.gguf b/SmallThinker-4BA0.6B-Instruct-IQ3_XS.gguf new file mode 100644 index 0000000..efbccd5 --- /dev/null +++ b/SmallThinker-4BA0.6B-Instruct-IQ3_XS.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ef377b427019ee5141e9d3b07a2e9698168b187d9f4d852996587ae28bb764fa +size 1849797696 diff --git a/SmallThinker-4BA0.6B-Instruct-IQ4_XS.gguf b/SmallThinker-4BA0.6B-Instruct-IQ4_XS.gguf new file mode 100644 index 0000000..6d0f6de --- /dev/null +++ b/SmallThinker-4BA0.6B-Instruct-IQ4_XS.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bfef04686803ce9e5e1479acb055075b70a84181cf9a1ef6eeddc0842550fa86 +size 2348799552 diff --git a/SmallThinker-4BA0.6B-Instruct-Q2_K.gguf b/SmallThinker-4BA0.6B-Instruct-Q2_K.gguf new file mode 100644 index 0000000..58ee790 --- /dev/null +++ b/SmallThinker-4BA0.6B-Instruct-Q2_K.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e23693a5e29f084e9b855e7fd4568ce58649ec2fa968a2bc413df17e9ba69c01 +size 1660846656 diff --git a/SmallThinker-4BA0.6B-Instruct-Q2_K_S.gguf b/SmallThinker-4BA0.6B-Instruct-Q2_K_S.gguf new file mode 100644 index 0000000..d7e9fe6 --- /dev/null +++ b/SmallThinker-4BA0.6B-Instruct-Q2_K_S.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2fd8d148fdb9af9ce0be291f85c7b0188c03d3e95cc2905068e9becd5166eb6b +size 1565884992 diff --git a/SmallThinker-4BA0.6B-Instruct-Q3_K_L.gguf b/SmallThinker-4BA0.6B-Instruct-Q3_K_L.gguf new file mode 100644 index 0000000..399b6a0 --- /dev/null +++ b/SmallThinker-4BA0.6B-Instruct-Q3_K_L.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:82a5532745e63ab9b54e1608de96c84e7ed077b3773988767a7a05c294129f72 +size 2271325248 diff --git a/SmallThinker-4BA0.6B-Instruct-Q3_K_M.gguf b/SmallThinker-4BA0.6B-Instruct-Q3_K_M.gguf new file mode 100644 index 0000000..3bf3873 --- /dev/null +++ b/SmallThinker-4BA0.6B-Instruct-Q3_K_M.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:633e8a10b62a50e16131d7560d7319ec6601d5faea3293927542d37b3ecfc6ad +size 2118855744 diff --git a/SmallThinker-4BA0.6B-Instruct-Q4_0.gguf b/SmallThinker-4BA0.6B-Instruct-Q4_0.gguf new file mode 100644 index 0000000..57a4883 --- /dev/null +++ b/SmallThinker-4BA0.6B-Instruct-Q4_0.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b3e622766dba7f196935cb2d5691209959a502229a1d43763a88d7e63f0aa26b +size 2482314816 diff --git a/SmallThinker-4BA0.6B-Instruct-Q4_K_M.gguf b/SmallThinker-4BA0.6B-Instruct-Q4_K_M.gguf new file mode 100644 index 0000000..2a3c4c4 --- /dev/null +++ b/SmallThinker-4BA0.6B-Instruct-Q4_K_M.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:df4014da836187ba5bfd586e3ff2dccc6c501368dc1ce0c64688b541cf3e9a41 +size 2630213184 diff --git a/SmallThinker-4BA0.6B-Instruct-Q4_K_S.gguf b/SmallThinker-4BA0.6B-Instruct-Q4_K_S.gguf new file mode 100644 index 0000000..b6c2edc --- /dev/null +++ b/SmallThinker-4BA0.6B-Instruct-Q4_K_S.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:783968d896f413c0c81e35b3cf526fe642741750c1e30aeab4b0ae5641b1c8fb +size 2491948608 diff --git a/SmallThinker-4BA0.6B-Instruct-Q5_0.gguf b/SmallThinker-4BA0.6B-Instruct-Q5_0.gguf new file mode 100644 index 0000000..cfd845c --- /dev/null +++ b/SmallThinker-4BA0.6B-Instruct-Q5_0.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:440be77dbc1c37494caeca4d6b8b5a306de2209847f1721d2269e25c1e08043b +size 2986491456 diff --git a/SmallThinker-4BA0.6B-Instruct-Q5_K_M.gguf b/SmallThinker-4BA0.6B-Instruct-Q5_K_M.gguf new file mode 100644 index 0000000..8f0579d --- /dev/null +++ b/SmallThinker-4BA0.6B-Instruct-Q5_K_M.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7201b9ee46b5f5621144f9eebd6fed29b00e326b1d8740be451091f2a9a6cece +size 3058105920 diff --git a/SmallThinker-4BA0.6B-Instruct-Q5_K_S.gguf b/SmallThinker-4BA0.6B-Instruct-Q5_K_S.gguf new file mode 100644 index 0000000..f4a4a72 --- /dev/null +++ b/SmallThinker-4BA0.6B-Instruct-Q5_K_S.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ac3090db698528b57a4193114b69d7a5a7d3ce0ba6d695b5b0173f5d8adaba35 +size 2977054272 diff --git a/SmallThinker-4BA0.6B-Instruct-Q6_K.gguf b/SmallThinker-4BA0.6B-Instruct-Q6_K.gguf new file mode 100644 index 0000000..e2dc1ca --- /dev/null +++ b/SmallThinker-4BA0.6B-Instruct-Q6_K.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1fa598626fbd322658c608ade2f34ed59113ff989385b51d5f4e24a84ab18461 +size 3512741952 diff --git a/SmallThinker-4BA0.6B-Instruct-Q8_0.gguf b/SmallThinker-4BA0.6B-Instruct-Q8_0.gguf new file mode 100644 index 0000000..241819b --- /dev/null +++ b/SmallThinker-4BA0.6B-Instruct-Q8_0.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b239d6c96d4e2599a5078bfd15c702e55fad31e4803e8304dbbb1b19ca1e4304 +size 4546104384