初始化项目，由ModelHub XC社区提供模型
Model: Mungert/MiniCPM4-0.5B-GGUF Source: Original Platform
This commit is contained in:
ModelHub XC
69
.gitattributes
vendored
Normal file
69
.gitattributes
vendored
Normal file
@@ -0,0 +1,69 @@
|
||||
*.7z filter=lfs diff=lfs merge=lfs -text
|
||||
*.arrow filter=lfs diff=lfs merge=lfs -text
|
||||
*.bin filter=lfs diff=lfs merge=lfs -text
|
||||
*.bin.* filter=lfs diff=lfs merge=lfs -text
|
||||
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
||||
*.ftz filter=lfs diff=lfs merge=lfs -text
|
||||
*.gz filter=lfs diff=lfs merge=lfs -text
|
||||
*.h5 filter=lfs diff=lfs merge=lfs -text
|
||||
*.joblib filter=lfs diff=lfs merge=lfs -text
|
||||
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
||||
*.model filter=lfs diff=lfs merge=lfs -text
|
||||
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
||||
*.onnx filter=lfs diff=lfs merge=lfs -text
|
||||
*.ot filter=lfs diff=lfs merge=lfs -text
|
||||
*.parquet filter=lfs diff=lfs merge=lfs -text
|
||||
*.pb filter=lfs diff=lfs merge=lfs -text
|
||||
*.pt filter=lfs diff=lfs merge=lfs -text
|
||||
*.pth filter=lfs diff=lfs merge=lfs -text
|
||||
*.rar filter=lfs diff=lfs merge=lfs -text
|
||||
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
||||
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
||||
*.tflite filter=lfs diff=lfs merge=lfs -text
|
||||
*.tgz filter=lfs diff=lfs merge=lfs -text
|
||||
*.xz filter=lfs diff=lfs merge=lfs -text
|
||||
*.zip filter=lfs diff=lfs merge=lfs -text
|
||||
*.zstandard filter=lfs diff=lfs merge=lfs -text
|
||||
*.tfevents* filter=lfs diff=lfs merge=lfs -text
|
||||
*.db* filter=lfs diff=lfs merge=lfs -text
|
||||
*.ark* filter=lfs diff=lfs merge=lfs -text
|
||||
**/*ckpt*data* filter=lfs diff=lfs merge=lfs -text
|
||||
**/*ckpt*.meta filter=lfs diff=lfs merge=lfs -text
|
||||
**/*ckpt*.index filter=lfs diff=lfs merge=lfs -text
|
||||
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
||||
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
||||
|
||||
*.ggml filter=lfs diff=lfs merge=lfs -text
|
||||
*.llamafile* filter=lfs diff=lfs merge=lfs -text
|
||||
*.pt2 filter=lfs diff=lfs merge=lfs -text
|
||||
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
||||
*.npy filter=lfs diff=lfs merge=lfs -text
|
||||
*.npz filter=lfs diff=lfs merge=lfs -text
|
||||
*.pickle filter=lfs diff=lfs merge=lfs -text
|
||||
*.pkl filter=lfs diff=lfs merge=lfs -text
|
||||
*.tar filter=lfs diff=lfs merge=lfs -text
|
||||
*.wasm filter=lfs diff=lfs merge=lfs -text
|
||||
*.zst filter=lfs diff=lfs merge=lfs -text
|
||||
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
||||
|
||||
MiniCPM4-0.5B-q4_k_s.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
MiniCPM4-0.5B-iq4_nl.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
MiniCPM4-0.5B-q6_k_m.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
MiniCPM4-0.5B-q3_k_m.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
MiniCPM4-0.5B-iq3_s.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
MiniCPM4-0.5B-iq3_xs.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
MiniCPM4-0.5B-iq4_xs.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
MiniCPM4-0.5B-q4_0.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
MiniCPM4-0.5B-iq3_xxs.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
MiniCPM4-0.5B-q5_k_s.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
MiniCPM4-0.5B-q5_k_m.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
MiniCPM4-0.5B-q5_0.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
MiniCPM4-0.5B-iq3_m.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
MiniCPM4-0.5B-q5_1.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
MiniCPM4-0.5B-f16_q8_0.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
MiniCPM4-0.5B-bf16_q8_0.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
MiniCPM4-0.5B-q4_1.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
MiniCPM4-0.5B-q3_k_s.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
MiniCPM4-0.5B-bf16.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
MiniCPM4-0.5B-q4_k_m.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
MiniCPM4-0.5B-q8_0.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
3
MiniCPM4-0.5B-bf16.gguf
Normal file
3
MiniCPM4-0.5B-bf16.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:3e31671966daac3ce360ff257c88b1e28a82db38f8b41a4bdaf2748711e2058b
|
||||
size 869538752
|
||||
3
MiniCPM4-0.5B-bf16_q8_0.gguf
Normal file
3
MiniCPM4-0.5B-bf16_q8_0.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:00d584450f3625d80b73eecacbc772a8bbf4c51ef748ce93dcc8ece33e578a79
|
||||
size 654252992
|
||||
3
MiniCPM4-0.5B-f16_q8_0.gguf
Normal file
3
MiniCPM4-0.5B-f16_q8_0.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:2f51ee44046c12e213f68d1706955dbb58e1ff06d77111888aee0ef0d6fef872
|
||||
size 654252992
|
||||
3
MiniCPM4-0.5B-iq3_m.gguf
Normal file
3
MiniCPM4-0.5B-iq3_m.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:b59717f4f5794179399232ea457a8dbbb2f20aa9adeb2649e9f17a9b4b9067b8
|
||||
size 210510592
|
||||
3
MiniCPM4-0.5B-iq3_s.gguf
Normal file
3
MiniCPM4-0.5B-iq3_s.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:d6e476c377bb209bd36cfbce336c23d6be589b2a31e9431b756393d944aac32a
|
||||
size 208003840
|
||||
3
MiniCPM4-0.5B-iq3_xs.gguf
Normal file
3
MiniCPM4-0.5B-iq3_xs.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:985a7220e4bc460ceebe5fcb2087c5473244c556ab12bbace7148b327fa4a952
|
||||
size 199598848
|
||||
3
MiniCPM4-0.5B-iq3_xxs.gguf
Normal file
3
MiniCPM4-0.5B-iq3_xxs.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:2d6354aa9d2320fb23146bf0eb9c8564a0e30ab38aa312c3d32d97bf3353621a
|
||||
size 191931136
|
||||
3
MiniCPM4-0.5B-iq4_nl.gguf
Normal file
3
MiniCPM4-0.5B-iq4_nl.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:25a6c370fa0078ad415417916d86853a8d3ae65b570646b501e389fb8f7fc677
|
||||
size 265700928
|
||||
3
MiniCPM4-0.5B-iq4_xs.gguf
Normal file
3
MiniCPM4-0.5B-iq4_xs.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:78289738856310da9170ca97189ccd92827f7fa027c2875c010d5d569e5e715c
|
||||
size 254592576
|
||||
3
MiniCPM4-0.5B-q3_k_m.gguf
Normal file
3
MiniCPM4-0.5B-q3_k_m.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:26d259936c1753f478440f6f27096c615ca8d60040946f0a5d6c19a2dbbce69f
|
||||
size 233801280
|
||||
3
MiniCPM4-0.5B-q3_k_s.gguf
Normal file
3
MiniCPM4-0.5B-q3_k_s.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:fa5e99142d8db366a4cfbe215d0a99c448134d15db5214b5da37ca6d88f6d45e
|
||||
size 210965248
|
||||
3
MiniCPM4-0.5B-q4_0.gguf
Normal file
3
MiniCPM4-0.5B-q4_0.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:4eaf28476c7f0e6cbfb6f310621e36df30b871d2fa29cf5f823a2f7f3bb25922
|
||||
size 245917440
|
||||
3
MiniCPM4-0.5B-q4_1.gguf
Normal file
3
MiniCPM4-0.5B-q4_1.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:61c7b9279d4df577f33ed822651f116f382d079cca178fe3c8664fffadb38c98
|
||||
size 273031424
|
||||
3
MiniCPM4-0.5B-q4_k_m.gguf
Normal file
3
MiniCPM4-0.5B-q4_k_m.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:66ef85bb806c973c3f24bb014b8bd2be4e41b5c51e2f64782f470589add87e74
|
||||
size 276028992
|
||||
3
MiniCPM4-0.5B-q4_k_s.gguf
Normal file
3
MiniCPM4-0.5B-q4_k_s.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:42e766396cde983de396a2c65c5d2cda13689203218e49c6bea0ca0c27c109ac
|
||||
size 269354560
|
||||
3
MiniCPM4-0.5B-q5_0.gguf
Normal file
3
MiniCPM4-0.5B-q5_0.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:cb56ed47d7a00d8fc4c2bd75088bafc7c9fbd8d7753f45295c6bee8bed9a3c35
|
||||
size 300145408
|
||||
3
MiniCPM4-0.5B-q5_1.gguf
Normal file
3
MiniCPM4-0.5B-q5_1.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:86558f4607cebdfc7c33eaf7689dc10395e17f8b61b5c83237c54e78b675bcf3
|
||||
size 327259392
|
||||
3
MiniCPM4-0.5B-q5_k_m.gguf
Normal file
3
MiniCPM4-0.5B-q5_k_m.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:391a5ba74df4465fbd1651f7098a3256da1b595bc6fb5edcad717a95dfaca738
|
||||
size 317863488
|
||||
3
MiniCPM4-0.5B-q5_k_s.gguf
Normal file
3
MiniCPM4-0.5B-q5_k_s.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:93a599070b4aeef23ee2b2fd8292957dcfa2af87e496c3b9ba806c34726fbf0d
|
||||
size 314416704
|
||||
3
MiniCPM4-0.5B-q6_k_m.gguf
Normal file
3
MiniCPM4-0.5B-q6_k_m.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:f9afc2e36168d9ce60d7a63d1b17f54ca9e8f2b29a8eaeb0d67b8d60386cbcef
|
||||
size 357762624
|
||||
3
MiniCPM4-0.5B-q8_0.gguf
Normal file
3
MiniCPM4-0.5B-q8_0.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:2501896738ad13a3eb5d4a6743f3f6ffed080a461cc1a8740856ff74598201f5
|
||||
size 462828992
|
||||
BIN
MiniCPM4-0.5B.imatrix
Normal file
BIN
MiniCPM4-0.5B.imatrix
Normal file
Binary file not shown.
311
README.md
Normal file
311
README.md
Normal file
@@ -0,0 +1,311 @@
|
||||
---
|
||||
license: apache-2.0
|
||||
language:
|
||||
- zh
|
||||
- en
|
||||
pipeline_tag: text-generation
|
||||
library_name: transformers
|
||||
---
|
||||
|
||||
# <span style="color: #7FFF7F;">MiniCPM4-0.5B GGUF Models</span>
|
||||
|
||||
|
||||
## <span style="color: #7F7FFF;">Model Generation Details</span>
|
||||
|
||||
This model was generated using [llama.cpp](https://github.com/ggerganov/llama.cpp) at commit [`6adc3c3e`](https://github.com/ggerganov/llama.cpp/commit/6adc3c3ebc029af058ac950a8e2a825fdf18ecc6).
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
---
|
||||
|
||||
<a href="https://readyforquantum.com/huggingface_gguf_selection_guide.html" style="color: #7FFF7F;">
|
||||
Click here to get info on choosing the right GGUF model format
|
||||
</a>
|
||||
|
||||
---
|
||||
|
||||
|
||||
|
||||
<!--Begin Original Model Card-->
|
||||
|
||||
<div align="center">
|
||||
<img src="https://github.com/OpenBMB/MiniCPM/blob/main/assets/minicpm_logo.png?raw=true" width="500em" ></img>
|
||||
</div>
|
||||
|
||||
<p align="center">
|
||||
<a href="https://github.com/OpenBMB/MiniCPM/" target="_blank">GitHub Repo</a> |
|
||||
<a href="https://github.com/OpenBMB/MiniCPM/tree/main/report/MiniCPM_4_Technical_Report.pdf" target="_blank">Technical Report</a>
|
||||
</p>
|
||||
<p align="center">
|
||||
👋 Join us on <a href="https://discord.gg/3cGQn9b3YM" target="_blank">Discord</a> and <a href="https://github.com/OpenBMB/MiniCPM/blob/main/assets/wechat.jpg" target="_blank">WeChat</a>
|
||||
</p>
|
||||
|
||||
## What's New
|
||||
- [2025.06.06] **MiniCPM4** series are released! This model achieves ultimate efficiency improvements while maintaining optimal performance at the same scale! It can achieve over 5x generation acceleration on typical end-side chips! You can find technical report [here](https://github.com/OpenBMB/MiniCPM/tree/main/report/MiniCPM_4_Technical_Report.pdf).🔥🔥🔥
|
||||
|
||||
## MiniCPM4 Series
|
||||
MiniCPM4 series are highly efficient large language models (LLMs) designed explicitly for end-side devices, which achieves this efficiency through systematic innovation in four key dimensions: model architecture, training data, training algorithms, and inference systems.
|
||||
- [MiniCPM4-8B](https://huggingface.co/openbmb/MiniCPM4-8B): The flagship of MiniCPM4, with 8B parameters, trained on 8T tokens.
|
||||
- [MiniCPM4-0.5B](https://huggingface.co/openbmb/MiniCPM4-0.5B): The small version of MiniCPM4, with 0.5B parameters, trained on 1T tokens. (**<-- you are here**)
|
||||
- [MiniCPM4-8B-Eagle-FRSpec](https://huggingface.co/openbmb/MiniCPM4-8B-Eagle-FRSpec): Eagle head for FRSpec, accelerating speculative inference for MiniCPM4-8B.
|
||||
- [MiniCPM4-8B-Eagle-FRSpec-QAT-cpmcu](https://huggingface.co/openbmb/MiniCPM4-8B-Eagle-FRSpec-QAT-cpmcu): Eagle head trained with QAT for FRSpec, efficiently integrate speculation and quantization to achieve ultra acceleration for MiniCPM4-8B.
|
||||
- [MiniCPM4-8B-Eagle-vLLM](https://huggingface.co/openbmb/MiniCPM4-8B-Eagle-vLLM): Eagle head in vLLM format, accelerating speculative inference for MiniCPM4-8B.
|
||||
- [MiniCPM4-8B-marlin-Eagle-vLLM](https://huggingface.co/openbmb/MiniCPM4-8B-marlin-Eagle-vLLM): Quantized Eagle head for vLLM format, accelerating speculative inference for MiniCPM4-8B.
|
||||
- [BitCPM4-0.5B](https://huggingface.co/openbmb/BitCPM4-0.5B): Extreme ternary quantization applied to MiniCPM4-0.5B compresses model parameters into ternary values, achieving a 90% reduction in bit width.
|
||||
- [BitCPM4-1B](https://huggingface.co/openbmb/BitCPM4-1B): Extreme ternary quantization applied to MiniCPM3-1B compresses model parameters into ternary values, achieving a 90% reduction in bit width.
|
||||
- [MiniCPM4-Survey](https://huggingface.co/openbmb/MiniCPM4-Survey): Based on MiniCPM4-8B, accepts users' quiries as input and autonomously generate trustworthy, long-form survey papers.
|
||||
- [MiniCPM4-MCP](https://huggingface.co/openbmb/MiniCPM4-MCP): Based on MiniCPM4-8B, accepts users' queries and available MCP tools as input and autonomously calls relevant MCP tools to satisfy users' requirements.
|
||||
|
||||
## Introduction
|
||||
MiniCPM 4 is an extremely efficient edge-side large model that has undergone efficient optimization across four dimensions: model architecture, learning algorithms, training data, and inference systems, achieving ultimate efficiency improvements.
|
||||
|
||||
- 🏗️ **Efficient Model Architecture:**
|
||||
- InfLLM v2 -- Trainable Sparse Attention Mechanism: Adopts a trainable sparse attention mechanism architecture where each token only needs to compute relevance with less than 5% of tokens in 128K long text processing, significantly reducing computational overhead for long texts
|
||||
|
||||
- 🧠 **Efficient Learning Algorithms:**
|
||||
- Model Wind Tunnel 2.0 -- Efficient Predictable Scaling: Introduces scaling prediction methods for performance of downstream tasks, enabling more precise model training configuration search
|
||||
- BitCPM -- Ultimate Ternary Quantization: Compresses model parameter bit-width to 3 values, achieving 90% extreme model bit-width reduction
|
||||
- Efficient Training Engineering Optimization: Adopts FP8 low-precision computing technology combined with Multi-token Prediction training strategy
|
||||
|
||||
- 📚 **High-Quality Training Data:**
|
||||
- UltraClean -- High-quality Pre-training Data Filtering and Generation: Builds iterative data cleaning strategies based on efficient data verification, open-sourcing high-quality Chinese and English pre-training dataset [UltraFinweb](https://huggingface.co/datasets/openbmb/Ultra-FineWeb)
|
||||
- UltraChat v2 -- High-quality Supervised Fine-tuning Data Generation: Constructs large-scale high-quality supervised fine-tuning datasets covering multiple dimensions including knowledge-intensive data, reasoning-intensive data, instruction-following data, long text understanding data, and tool calling data
|
||||
|
||||
- ⚡ **Efficient Inference System:**
|
||||
- CPM.cu -- Lightweight and Efficient CUDA Inference Framework: Integrates sparse attention, model quantization, and speculative sampling to achieve efficient prefilling and decoding
|
||||
- ArkInfer -- Cross-platform Deployment System: Supports efficient deployment across multiple backend environments, providing flexible cross-platform adaptation capabilities
|
||||
|
||||
## Usage
|
||||
### Inference with Transformers
|
||||
```python
|
||||
from transformers import AutoModelForCausalLM, AutoTokenizer
|
||||
import torch
|
||||
torch.manual_seed(0)
|
||||
|
||||
path = 'openbmb/MiniCPM4-0.5B'
|
||||
device = "cuda"
|
||||
tokenizer = AutoTokenizer.from_pretrained(path)
|
||||
model = AutoModelForCausalLM.from_pretrained(path, torch_dtype=torch.bfloat16, device_map=device, trust_remote_code=True)
|
||||
|
||||
# User can directly use the chat interface
|
||||
responds, history = model.chat(tokenizer, "Write an article about Artificial Intelligence.", temperature=0.7, top_p=0.7)
|
||||
print(responds)
|
||||
|
||||
# User can also use the generate interface
|
||||
# messages = [
|
||||
# {"role": "user", "content": "Write an article about Artificial Intelligence."},
|
||||
# ]
|
||||
# prompt_text = tokenizer.apply_chat_template(
|
||||
# messages,
|
||||
# tokenize=False,
|
||||
# add_generation_prompt=True,
|
||||
# )
|
||||
# model_inputs = tokenizer([prompt_text], return_tensors="pt").to(device)
|
||||
|
||||
# model_outputs = model.generate(
|
||||
# **model_inputs,
|
||||
# max_new_tokens=1024,
|
||||
# top_p=0.7,
|
||||
# temperature=0.7
|
||||
# )
|
||||
# output_token_ids = [
|
||||
# model_outputs[i][len(model_inputs[i]):] for i in range(len(model_inputs['input_ids']))
|
||||
# ]
|
||||
|
||||
# responses = tokenizer.batch_decode(output_token_ids, skip_special_tokens=True)[0]
|
||||
# print(responses)
|
||||
```
|
||||
|
||||
### Inference with [SGLang](https://github.com/sgl-project/sglang)
|
||||
|
||||
For now, you need to install our forked version of SGLang.
|
||||
```bash
|
||||
git clone -b openbmb https://github.com/OpenBMB/sglang.git
|
||||
cd sglang
|
||||
|
||||
pip install --upgrade pip
|
||||
pip install -e "python[all]"
|
||||
```
|
||||
|
||||
You can start the inference server by running the following command:
|
||||
```bash
|
||||
python -m sglang.launch_server --model openbmb/MiniCPM4-0.5B --trust-remote-code --port 30000 --chat-template chatml
|
||||
```
|
||||
|
||||
Then you can use the chat interface by running the following command:
|
||||
```python
|
||||
import openai
|
||||
|
||||
client = openai.Client(base_url=f"http://localhost:30000/v1", api_key="None")
|
||||
|
||||
response = client.chat.completions.create(
|
||||
model="openbmb/MiniCPM4-0.5B",
|
||||
messages=[
|
||||
{"role": "user", "content": "Write an article about Artificial Intelligence."},
|
||||
],
|
||||
temperature=0.7,
|
||||
max_tokens=1024,
|
||||
)
|
||||
|
||||
print(response.choices[0].message.content)
|
||||
```
|
||||
|
||||
### Inference with [vLLM](https://github.com/vllm-project/vllm)
|
||||
For now, you need to install the latest version of vLLM.
|
||||
```
|
||||
pip install -U vllm \
|
||||
--pre \
|
||||
--extra-index-url https://wheels.vllm.ai/nightly
|
||||
```
|
||||
|
||||
Then you can inference MiniCPM4-0.5B with vLLM:
|
||||
```python
|
||||
from transformers import AutoTokenizer
|
||||
from vllm import LLM, SamplingParams
|
||||
|
||||
model_name = "openbmb/MiniCPM4-0.5B"
|
||||
prompt = [{"role": "user", "content": "Please recommend 5 tourist attractions in Beijing. "}]
|
||||
|
||||
tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
|
||||
input_text = tokenizer.apply_chat_template(prompt, tokenize=False, add_generation_prompt=True)
|
||||
|
||||
llm = LLM(
|
||||
model=model_name,
|
||||
trust_remote_code=True,
|
||||
max_num_batched_tokens=32768,
|
||||
dtype="bfloat16",
|
||||
gpu_memory_utilization=0.8,
|
||||
)
|
||||
sampling_params = SamplingParams(top_p=0.7, temperature=0.7, max_tokens=1024, repetition_penalty=1.02)
|
||||
|
||||
outputs = llm.generate(prompts=input_text, sampling_params=sampling_params)
|
||||
|
||||
print(outputs[0].outputs[0].text)
|
||||
```
|
||||
|
||||
Also, you can start the inference server by running the following command:
|
||||
> **Note**: In vLLM's chat API, `add_special_tokens` is `False` by default. This means important special tokens—such as the beginning-of-sequence (BOS) token—will not be added automatically. To ensure the input prompt is correctly formatted for the model, you should explicitly set `extra_body={"add_special_tokens": True}`.
|
||||
|
||||
```bash
|
||||
vllm serve openbmb/MiniCPM4-0.5B
|
||||
```
|
||||
|
||||
Then you can use the chat interface by running the following code:
|
||||
|
||||
```python
|
||||
import openai
|
||||
|
||||
client = openai.Client(base_url="http://localhost:8000/v1", api_key="EMPTY")
|
||||
|
||||
response = client.chat.completions.create(
|
||||
model="openbmb/MiniCPM4-0.5B",
|
||||
messages=[
|
||||
{"role": "user", "content": "Write an article about Artificial Intelligence."},
|
||||
],
|
||||
temperature=0.7,
|
||||
max_tokens=1024,
|
||||
extra_body=dict(add_special_tokens=True), # Ensures special tokens are added for chat template
|
||||
|
||||
)
|
||||
|
||||
print(response.choices[0].message.content)
|
||||
```
|
||||
|
||||
|
||||
## Evaluation Results
|
||||
On two typical end-side chips, Jetson AGX Orin and RTX 4090, MiniCPM4 demonstrates significantly faster processing speed compared to similar-size models in long text processing tasks. As text length increases, MiniCPM4's efficiency advantage becomes more pronounced. On the Jetson AGX Orin platform, compared to Qwen3-8B, MiniCPM4 achieves approximately 7x decoding speed improvement.
|
||||
|
||||
|
||||
|
||||
#### Comprehensive Evaluation
|
||||
MiniCPM4 launches end-side versions with 8B and 0.5B parameter scales, both achieving best-in-class performance in their respective categories.
|
||||
|
||||
|
||||
|
||||
#### Long Text Evaluation
|
||||
MiniCPM4 is pre-trained on 32K long texts and achieves length extension through YaRN technology. In the 128K long text needle-in-a-haystack task, MiniCPM4 demonstrates outstanding performance.
|
||||
|
||||
|
||||
|
||||
## Statement
|
||||
- As a language model, MiniCPM generates content by learning from a vast amount of text.
|
||||
- However, it does not possess the ability to comprehend or express personal opinions or value judgments.
|
||||
- Any content generated by MiniCPM does not represent the viewpoints or positions of the model developers.
|
||||
- Therefore, when using content generated by MiniCPM, users should take full responsibility for evaluating and verifying it on their own.
|
||||
|
||||
## LICENSE
|
||||
- This repository and MiniCPM models are released under the [Apache-2.0](https://github.com/OpenBMB/MiniCPM/blob/main/LICENSE) License.
|
||||
|
||||
## Citation
|
||||
- Please cite our [paper](https://github.com/OpenBMB/MiniCPM/tree/main/report/MiniCPM_4_Technical_Report.pdf) if you find our work valuable.
|
||||
|
||||
```bibtex
|
||||
@article{minicpm4,
|
||||
title={{MiniCPM4}: Ultra-Efficient LLMs on End Devices},
|
||||
author={MiniCPM Team},
|
||||
year={2025}
|
||||
}
|
||||
```
|
||||
|
||||
|
||||
<!--End Original Model Card-->
|
||||
|
||||
---
|
||||
|
||||
# <span id="testllm" style="color: #7F7FFF;">🚀 If you find these models useful</span>
|
||||
|
||||
Help me test my **AI-Powered Quantum Network Monitor Assistant** with **quantum-ready security checks**:
|
||||
|
||||
👉 [Quantum Network Monitor](https://readyforquantum.com/?assistant=open&utm_source=huggingface&utm_medium=referral&utm_campaign=huggingface_repo_readme)
|
||||
|
||||
|
||||
The full Open Source Code for the Quantum Network Monitor Service available at my github repos ( repos with NetworkMonitor in the name) : [Source Code Quantum Network Monitor](https://github.com/Mungert69). You will also find the code I use to quantize the models if you want to do it yourself [GGUFModelBuilder](https://github.com/Mungert69/GGUFModelBuilder)
|
||||
|
||||
💬 **How to test**:
|
||||
Choose an **AI assistant type**:
|
||||
- `TurboLLM` (GPT-4.1-mini)
|
||||
- `HugLLM` (Hugginface Open-source models)
|
||||
- `TestLLM` (Experimental CPU-only)
|
||||
|
||||
### **What I’m Testing**
|
||||
I’m pushing the limits of **small open-source models for AI network monitoring**, specifically:
|
||||
- **Function calling** against live network services
|
||||
- **How small can a model go** while still handling:
|
||||
- Automated **Nmap security scans**
|
||||
- **Quantum-readiness checks**
|
||||
- **Network Monitoring tasks**
|
||||
|
||||
🟡 **TestLLM** – Current experimental model (llama.cpp on 2 CPU threads on huggingface docker space):
|
||||
- ✅ **Zero-configuration setup**
|
||||
- ⏳ 30s load time (slow inference but **no API costs**) . No token limited as the cost is low.
|
||||
- 🔧 **Help wanted!** If you’re into **edge-device AI**, let’s collaborate!
|
||||
|
||||
### **Other Assistants**
|
||||
🟢 **TurboLLM** – Uses **gpt-4.1-mini** :
|
||||
- **It performs very well but unfortunatly OpenAI charges per token. For this reason tokens usage is limited.
|
||||
- **Create custom cmd processors to run .net code on Quantum Network Monitor Agents**
|
||||
- **Real-time network diagnostics and monitoring**
|
||||
- **Security Audits**
|
||||
- **Penetration testing** (Nmap/Metasploit)
|
||||
|
||||
🔵 **HugLLM** – Latest Open-source models:
|
||||
- 🌐 Runs on Hugging Face Inference API. Performs pretty well using the lastest models hosted on Novita.
|
||||
|
||||
### 💡 **Example commands you could test**:
|
||||
1. `"Give me info on my websites SSL certificate"`
|
||||
2. `"Check if my server is using quantum safe encyption for communication"`
|
||||
3. `"Run a comprehensive security audit on my server"`
|
||||
4. '"Create a cmd processor to .. (what ever you want)" Note you need to install a [Quantum Network Monitor Agent](https://readyforquantum.com/Download/?utm_source=huggingface&utm_medium=referral&utm_campaign=huggingface_repo_readme) to run the .net code on. This is a very flexible and powerful feature. Use with caution!
|
||||
|
||||
### Final Word
|
||||
|
||||
I fund the servers used to create these model files, run the Quantum Network Monitor service, and pay for inference from Novita and OpenAI—all out of my own pocket. All the code behind the model creation and the Quantum Network Monitor project is [open source](https://github.com/Mungert69). Feel free to use whatever you find helpful.
|
||||
|
||||
If you appreciate the work, please consider [buying me a coffee](https://www.buymeacoffee.com/mahadeva) ☕. Your support helps cover service costs and allows me to raise token limits for everyone.
|
||||
|
||||
I'm also open to job opportunities or sponsorship.
|
||||
|
||||
Thank you! 😊
|
||||
1
configuration.json
Normal file
1
configuration.json
Normal file
@@ -0,0 +1 @@
|
||||
{"framework": "pytorch", "task": "text-generation", "allow_remote": true}
|
||||
Reference in New Issue
Block a user