commit 57aa45b15901adfcfc3c5d0ab264fcad50f5851f Author: ModelHub XC Date: Sun Jul 12 03:52:06 2026 +0800 初始化项目，由ModelHub XC社区提供模型 Model: unsloth/ERNIE-4.5-21B-A3B-Thinking-GGUF Source: Original Platform diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..4268c80 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,76 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bin.* filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text + +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zstandard filter=lfs diff=lfs merge=lfs -text +*.tfevents* filter=lfs diff=lfs merge=lfs -text +*.db* filter=lfs diff=lfs merge=lfs -text +*.ark* filter=lfs diff=lfs merge=lfs -text +**/*ckpt*data* filter=lfs diff=lfs merge=lfs -text +**/*ckpt*.meta filter=lfs diff=lfs merge=lfs -text +**/*ckpt*.index filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text + +*.ggml filter=lfs diff=lfs merge=lfs -text +*.llamafile* filter=lfs diff=lfs merge=lfs -text +*.pt2 filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text + +ERNIE-4.5-21B-A3B-Thinking-UD-IQ1_S.gguf filter=lfs diff=lfs merge=lfs -text +ERNIE-4.5-21B-A3B-Thinking-Q3_K_M.gguf filter=lfs diff=lfs merge=lfs -text +ERNIE-4.5-21B-A3B-Thinking-UD-IQ2_M.gguf filter=lfs diff=lfs merge=lfs -text +ERNIE-4.5-21B-A3B-Thinking-UD-Q8_K_XL.gguf filter=lfs diff=lfs merge=lfs -text +ERNIE-4.5-21B-A3B-Thinking-Q4_1.gguf filter=lfs diff=lfs merge=lfs -text +ERNIE-4.5-21B-A3B-Thinking-Q2_K_L.gguf filter=lfs diff=lfs merge=lfs -text +ERNIE-4.5-21B-A3B-Thinking-UD-IQ1_M.gguf filter=lfs diff=lfs merge=lfs -text +ERNIE-4.5-21B-A3B-Thinking-Q3_K_S.gguf filter=lfs diff=lfs merge=lfs -text +ERNIE-4.5-21B-A3B-Thinking-Q8_0.gguf filter=lfs diff=lfs merge=lfs -text +ERNIE-4.5-21B-A3B-Thinking-UD-IQ2_XXS.gguf filter=lfs diff=lfs merge=lfs -text +ERNIE-4.5-21B-A3B-Thinking-Q2_K.gguf filter=lfs diff=lfs merge=lfs -text +ERNIE-4.5-21B-A3B-Thinking-Q4_0.gguf filter=lfs diff=lfs merge=lfs -text +ERNIE-4.5-21B-A3B-Thinking-UD-Q2_K_XL.gguf filter=lfs diff=lfs merge=lfs -text +ERNIE-4.5-21B-A3B-Thinking-Q4_K_M.gguf filter=lfs diff=lfs merge=lfs -text +ERNIE-4.5-21B-A3B-Thinking-Q6_K.gguf filter=lfs diff=lfs merge=lfs -text +ERNIE-4.5-21B-A3B-Thinking-BF16.gguf filter=lfs diff=lfs merge=lfs -text +ERNIE-4.5-21B-A3B-Thinking-UD-Q6_K_XL.gguf filter=lfs diff=lfs merge=lfs -text +imatrix_unsloth.gguf_file filter=lfs diff=lfs merge=lfs -text +ERNIE-4.5-21B-A3B-Thinking-UD-IQ3_XXS.gguf filter=lfs diff=lfs merge=lfs -text +ERNIE-4.5-21B-A3B-Thinking-UD-TQ1_0.gguf filter=lfs diff=lfs merge=lfs -text +ERNIE-4.5-21B-A3B-Thinking-UD-Q3_K_XL.gguf filter=lfs diff=lfs merge=lfs -text +ERNIE-4.5-21B-A3B-Thinking-UD-Q4_K_XL.gguf filter=lfs diff=lfs merge=lfs -text +ERNIE-4.5-21B-A3B-Thinking-Q5_K_S.gguf filter=lfs diff=lfs merge=lfs -text +ERNIE-4.5-21B-A3B-Thinking-Q5_K_M.gguf filter=lfs diff=lfs merge=lfs -text +ERNIE-4.5-21B-A3B-Thinking-IQ4_XS.gguf filter=lfs diff=lfs merge=lfs -text +ERNIE-4.5-21B-A3B-Thinking-Q4_K_S.gguf filter=lfs diff=lfs merge=lfs -text +ERNIE-4.5-21B-A3B-Thinking-IQ4_NL.gguf filter=lfs diff=lfs merge=lfs -text +ERNIE-4.5-21B-A3B-Thinking-UD-Q5_K_XL.gguf filter=lfs diff=lfs merge=lfs -text \ No newline at end of file diff --git a/ERNIE-4.5-21B-A3B-Thinking-BF16.gguf b/ERNIE-4.5-21B-A3B-Thinking-BF16.gguf new file mode 100644 index 0000000..f0914b4 --- /dev/null +++ b/ERNIE-4.5-21B-A3B-Thinking-BF16.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1334cca26311af9c5e312f7edf57356ed8ffe6790a70a71f4ddc09408003d979 +size 43662420800 diff --git a/ERNIE-4.5-21B-A3B-Thinking-IQ4_NL.gguf b/ERNIE-4.5-21B-A3B-Thinking-IQ4_NL.gguf new file mode 100644 index 0000000..0232d58 --- /dev/null +++ b/ERNIE-4.5-21B-A3B-Thinking-IQ4_NL.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:61ab23f67fe69af34e7c352dcaa5db4289886f4083767ec9af7e600fea2f11e3 +size 12475599744 diff --git a/ERNIE-4.5-21B-A3B-Thinking-IQ4_XS.gguf b/ERNIE-4.5-21B-A3B-Thinking-IQ4_XS.gguf new file mode 100644 index 0000000..e5bf592 --- /dev/null +++ b/ERNIE-4.5-21B-A3B-Thinking-IQ4_XS.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:26ee8fc7dd3f16913d76afda8e57ac3fdff8a0da14c4ad34cbbe96d6a9b34a62 +size 11823025024 diff --git a/ERNIE-4.5-21B-A3B-Thinking-Q2_K.gguf b/ERNIE-4.5-21B-A3B-Thinking-Q2_K.gguf new file mode 100644 index 0000000..b93a435 --- /dev/null +++ b/ERNIE-4.5-21B-A3B-Thinking-Q2_K.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:16989cfc50577b03621bc3f9d45b38bb231b209bba4e5e4e109e33b275fe735e +size 8152599424 diff --git a/ERNIE-4.5-21B-A3B-Thinking-Q2_K_L.gguf b/ERNIE-4.5-21B-A3B-Thinking-Q2_K_L.gguf new file mode 100644 index 0000000..b93a435 --- /dev/null +++ b/ERNIE-4.5-21B-A3B-Thinking-Q2_K_L.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:16989cfc50577b03621bc3f9d45b38bb231b209bba4e5e4e109e33b275fe735e +size 8152599424 diff --git a/ERNIE-4.5-21B-A3B-Thinking-Q3_K_M.gguf b/ERNIE-4.5-21B-A3B-Thinking-Q3_K_M.gguf new file mode 100644 index 0000000..33da99d --- /dev/null +++ b/ERNIE-4.5-21B-A3B-Thinking-Q3_K_M.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f450a661dd2047635eb41e5cbac4865f142b44c7d28824453598348d6a537c96 +size 10524818304 diff --git a/ERNIE-4.5-21B-A3B-Thinking-Q3_K_S.gguf b/ERNIE-4.5-21B-A3B-Thinking-Q3_K_S.gguf new file mode 100644 index 0000000..1e59fcd --- /dev/null +++ b/ERNIE-4.5-21B-A3B-Thinking-Q3_K_S.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c9cc8ceee654b9bb4ce49cfdbcc7bfbc1472e488e1a8c32b909eda2b2139e88f +size 9639611264 diff --git a/ERNIE-4.5-21B-A3B-Thinking-Q4_0.gguf b/ERNIE-4.5-21B-A3B-Thinking-Q4_0.gguf new file mode 100644 index 0000000..df32a45 --- /dev/null +++ b/ERNIE-4.5-21B-A3B-Thinking-Q4_0.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:133373bbbc4c94801ca2dc2a59bfe0ebc95de654167c402e9bf5aaed5db2bc00 +size 12501363584 diff --git a/ERNIE-4.5-21B-A3B-Thinking-Q4_1.gguf b/ERNIE-4.5-21B-A3B-Thinking-Q4_1.gguf new file mode 100644 index 0000000..4554562 --- /dev/null +++ b/ERNIE-4.5-21B-A3B-Thinking-Q4_1.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0bafca504e2ebe5b1832904d98da16fef6ee9c7ef72eb797f6047eef5cae37fd +size 13778455424 diff --git a/ERNIE-4.5-21B-A3B-Thinking-Q4_K_M.gguf b/ERNIE-4.5-21B-A3B-Thinking-Q4_K_M.gguf new file mode 100644 index 0000000..96bd2d4 --- /dev/null +++ b/ERNIE-4.5-21B-A3B-Thinking-Q4_K_M.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9cd7c60d3c30f46e7f539043ec19c7ec8953153f65dd9a2a2748b8e3f589d015 +size 13331018624 diff --git a/ERNIE-4.5-21B-A3B-Thinking-Q4_K_S.gguf b/ERNIE-4.5-21B-A3B-Thinking-Q4_K_S.gguf new file mode 100644 index 0000000..596fe9f --- /dev/null +++ b/ERNIE-4.5-21B-A3B-Thinking-Q4_K_S.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ed79f0dd3f22a21c23a4d2d0419be8dade51c48671d8616b6d9661c7b7d4366b +size 12536097664 diff --git a/ERNIE-4.5-21B-A3B-Thinking-Q5_K_M.gguf b/ERNIE-4.5-21B-A3B-Thinking-Q5_K_M.gguf new file mode 100644 index 0000000..a24b06c --- /dev/null +++ b/ERNIE-4.5-21B-A3B-Thinking-Q5_K_M.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5e6c6348482179059ad5b9494125b36f77b5f8fcfeaf86352ec78fafa93e0be7 +size 15585334144 diff --git a/ERNIE-4.5-21B-A3B-Thinking-Q5_K_S.gguf b/ERNIE-4.5-21B-A3B-Thinking-Q5_K_S.gguf new file mode 100644 index 0000000..2d6a9ed --- /dev/null +++ b/ERNIE-4.5-21B-A3B-Thinking-Q5_K_S.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0a6d9145fa967b52bd4ca5507522ce94021d4ea920ba21ae8d69a5b85f63ca6c +size 15142300544 diff --git a/ERNIE-4.5-21B-A3B-Thinking-Q6_K.gguf b/ERNIE-4.5-21B-A3B-Thinking-Q6_K.gguf new file mode 100644 index 0000000..52a64bd --- /dev/null +++ b/ERNIE-4.5-21B-A3B-Thinking-Q6_K.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e471e9cf3563a83b6a26a8f0342993f62e8d103d5695a9d221fc9f9a59d0bb88 +size 17972042624 diff --git a/ERNIE-4.5-21B-A3B-Thinking-Q8_0.gguf b/ERNIE-4.5-21B-A3B-Thinking-Q8_0.gguf new file mode 100644 index 0000000..c473e13 --- /dev/null +++ b/ERNIE-4.5-21B-A3B-Thinking-Q8_0.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3347f8e6028c7722dd94491558eb97622f62a4b2a6a54f269bff669c87470fd0 +size 23205358464 diff --git a/ERNIE-4.5-21B-A3B-Thinking-UD-IQ1_M.gguf b/ERNIE-4.5-21B-A3B-Thinking-UD-IQ1_M.gguf new file mode 100644 index 0000000..3d37c52 --- /dev/null +++ b/ERNIE-4.5-21B-A3B-Thinking-UD-IQ1_M.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9fea424658d7f62669defd86d99e810e14bf0ee2fb33f150cf37ba11243dc93e +size 7243082624 diff --git a/ERNIE-4.5-21B-A3B-Thinking-UD-IQ1_S.gguf b/ERNIE-4.5-21B-A3B-Thinking-UD-IQ1_S.gguf new file mode 100644 index 0000000..542b412 --- /dev/null +++ b/ERNIE-4.5-21B-A3B-Thinking-UD-IQ1_S.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4d4f046d7fc33aa73586eedd9e94f2f8878518753f9a118ea551b2b88213a661 +size 7010470784 diff --git a/ERNIE-4.5-21B-A3B-Thinking-UD-IQ2_M.gguf b/ERNIE-4.5-21B-A3B-Thinking-UD-IQ2_M.gguf new file mode 100644 index 0000000..5972570 --- /dev/null +++ b/ERNIE-4.5-21B-A3B-Thinking-UD-IQ2_M.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:47eb16e2364b975a69e0ee764a7c2666abc2d6d7d294afbab5b4a2d1c990911b +size 8033405824 diff --git a/ERNIE-4.5-21B-A3B-Thinking-UD-IQ2_XXS.gguf b/ERNIE-4.5-21B-A3B-Thinking-UD-IQ2_XXS.gguf new file mode 100644 index 0000000..1d5fb44 --- /dev/null +++ b/ERNIE-4.5-21B-A3B-Thinking-UD-IQ2_XXS.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1387693f9ad00c915a4283094205a6d81bf89c09448afa9ffa4d4e1957580b14 +size 7767206784 diff --git a/ERNIE-4.5-21B-A3B-Thinking-UD-IQ3_XXS.gguf b/ERNIE-4.5-21B-A3B-Thinking-UD-IQ3_XXS.gguf new file mode 100644 index 0000000..f6e7bad --- /dev/null +++ b/ERNIE-4.5-21B-A3B-Thinking-UD-IQ3_XXS.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:75a729aef2114bb140ca374d635a2fd6747f04b9d69903a6d631b4dc017e6f71 +size 9548209024 diff --git a/ERNIE-4.5-21B-A3B-Thinking-UD-Q2_K_XL.gguf b/ERNIE-4.5-21B-A3B-Thinking-UD-Q2_K_XL.gguf new file mode 100644 index 0000000..c63837c --- /dev/null +++ b/ERNIE-4.5-21B-A3B-Thinking-UD-Q2_K_XL.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:840c34e6080736073bc8ece513aae8c2a0f49af971d41b3ece8d9fda0e4c0fb2 +size 8222886784 diff --git a/ERNIE-4.5-21B-A3B-Thinking-UD-Q3_K_XL.gguf b/ERNIE-4.5-21B-A3B-Thinking-UD-Q3_K_XL.gguf new file mode 100644 index 0000000..8959435 --- /dev/null +++ b/ERNIE-4.5-21B-A3B-Thinking-UD-Q3_K_XL.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d62562ddbf728925a4d233fe44a192c71b671ae742caae1ae245e5f3462eee8c +size 10577165184 diff --git a/ERNIE-4.5-21B-A3B-Thinking-UD-Q4_K_XL.gguf b/ERNIE-4.5-21B-A3B-Thinking-UD-Q4_K_XL.gguf new file mode 100644 index 0000000..a0c8431 --- /dev/null +++ b/ERNIE-4.5-21B-A3B-Thinking-UD-Q4_K_XL.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:88a031e033df34c30c9d46cf3b68773f3638dacb6550b80b8940f947160bb93e +size 13400773504 diff --git a/ERNIE-4.5-21B-A3B-Thinking-UD-Q5_K_XL.gguf b/ERNIE-4.5-21B-A3B-Thinking-UD-Q5_K_XL.gguf new file mode 100644 index 0000000..9faa32e --- /dev/null +++ b/ERNIE-4.5-21B-A3B-Thinking-UD-Q5_K_XL.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d69e0dcbef27f09074ded596943be8fa70f2296a29c54de7a8d464c39a46dbb4 +size 15676849024 diff --git a/ERNIE-4.5-21B-A3B-Thinking-UD-Q6_K_XL.gguf b/ERNIE-4.5-21B-A3B-Thinking-UD-Q6_K_XL.gguf new file mode 100644 index 0000000..ccc93a5 --- /dev/null +++ b/ERNIE-4.5-21B-A3B-Thinking-UD-Q6_K_XL.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:af12c93ef2a446a62ce814291bd6c4536c7156f9ea629268617c79da12e01f26 +size 19260214144 diff --git a/ERNIE-4.5-21B-A3B-Thinking-UD-Q8_K_XL.gguf b/ERNIE-4.5-21B-A3B-Thinking-UD-Q8_K_XL.gguf new file mode 100644 index 0000000..1d14d91 --- /dev/null +++ b/ERNIE-4.5-21B-A3B-Thinking-UD-Q8_K_XL.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d6eb94571d5ea046e9a24a4e13f46815c5dda8e0a75efcb9c4991e1f2a430dd8 +size 26643540864 diff --git a/ERNIE-4.5-21B-A3B-Thinking-UD-TQ1_0.gguf b/ERNIE-4.5-21B-A3B-Thinking-UD-TQ1_0.gguf new file mode 100644 index 0000000..be8966e --- /dev/null +++ b/ERNIE-4.5-21B-A3B-Thinking-UD-TQ1_0.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9964fc2ddeafaa8fe62f6a1f26d369cdbeca4e35490aabe01fffbb54bf024aff +size 6497323904 diff --git a/README.md b/README.md new file mode 100644 index 0000000..7b76cc1 --- /dev/null +++ b/README.md @@ -0,0 +1,210 @@ +--- +license: apache-2.0 +language: +- en +- zh +pipeline_tag: text-generation +tags: +- ERNIE4.5 +library_name: transformers +base_model: baidu/ERNIE-4.5-21B-A3B-Thinking +--- + +
+ + Chat + + + Hugging Face + + + Github + + + Blog + + + Discord + + + X + +
+ +
+ + License + +
+ +# ERNIE-4.5-21B-A3B-Thinking + +## Model Highlights + +Over the past three months, we have continued to scale the **thinking capability** of ERNIE-4.5-21B-A3B, improving both the **quality and depth** of reasoning, thereby advancing the competitiveness of ERNIE **lightweight models** in complex reasoning tasks. We are pleased to introduce **ERNIE-4.5-21B-A3B-Thinking**, featuring the following key enhancements: + +* **Significantly improved performance** on reasoning tasks, including logical reasoning, mathematics, science, coding, text generation, and academic benchmarks that typically require human expertise. +* **Efficient tool usage** capabilities. +* **Enhanced 128K long-context understanding** capabilities. + +> [!NOTE] +> Note: This version has an increased thinking length. We strongly recommend its use in highly complex reasoning tasks. + +![benchmark](./benchmark.png) + +## Model Overview + +ERNIE-4.5-21B-A3B-Thinking is a text MoE post-trained model, with 21B total parameters and 3B activated parameters for each token. The following are the model configuration details: + +|Key|Value| +|-|-| +|Modality|Text| +|Training Stage|Posttraining| +|Params(Total / Activated)|21B / 3B| +|Layers|28| +|Heads(Q/KV)|20 / 4| +|Text Experts(Total / Activated)|64 / 6| +|Vision Experts(Total / Activated)|64 / 6| +|Shared Experts|2| +|Context Length|131072| + +## Quickstart + +> [!NOTE] +> To align with the wider community, this model releases Transformer-style weights. Both PyTorch and PaddlePaddle ecosystem tools, such as vLLM, transformers, and FastDeploy, are expected to be able to load and run this model. + +### FastDeploy Inference + +Quickly deploy services using FastDeploy as shown below. For more detailed usage, refer to the [FastDeploy GitHub Repository](https://github.com/PaddlePaddle/FastDeploy). + +**Note**: 80GB x 1 GPU resources are required. Deploying this model requires FastDeploy version 2.2. + +```bash +python -m fastdeploy.entrypoints.openai.api_server \ + --model baidu/ERNIE-4.5-21B-A3B-Thinking \ + --port 8180 \ + --metrics-port 8181 \ + --engine-worker-queue-port 8182 \ + --load_choices "default_v1" \ + --tensor-parallel-size 1 \ + --max-model-len 131072 \ + --reasoning-parser ernie_x1 \ + --tool-call-parser ernie_x1 \ + --max-num-seqs 32 +``` + +The ERNIE-4.5-21B-A3B-Thinking model supports function call. + +```bash +curl -X POST "http://0.0.0.0:8180/v1/chat/completions" \ +-H "Content-Type: application/json" \ +-d $'{ + "messages": [ + { + "role": "user", + "content": "How \'s the weather in Beijing today?" + } + ], + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Determine weather in my location", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state e.g. San Francisco, CA" + }, + "unit": { + "type": "string", + "enum": [ + "c", + "f" + ] + } + }, + "additionalProperties": false, + "required": [ + "location", + "unit" + ] + }, + "strict": true + } + }] +}' +``` + +### vLLM inference + +```bash +vllm serve baidu/ERNIE-4.5-21B-A3B-Thinking +``` + +The `reasoning-parser` and `tool-call-parser` for vLLM Ernie are currently under development. + +### Using `transformers` library + +**Note**: You'll need the`transformers`library (version 4.54.0 or newer) installed to use this model. + +The following contains a code snippet illustrating how to use the model generate content based on given inputs. + +```python +import torch +from transformers import AutoModelForCausalLM, AutoTokenizer + +model_name = "baidu/ERNIE-4.5-21B-A3B-Thinking" + +# load the tokenizer and the model +tokenizer = AutoTokenizer.from_pretrained(model_name) +model = AutoModelForCausalLM.from_pretrained( + model_name, + device_map="auto", + torch_dtype=torch.bfloat16, +) + +# prepare the model input +prompt = "Give me a short introduction to large language model." +messages = [ + {"role": "user", "content": prompt} +] +text = tokenizer.apply_chat_template( + messages, + tokenize=False, + add_generation_prompt=True +) +model_inputs = tokenizer([text], add_special_tokens=False, return_tensors="pt").to(model.device) + +# conduct text completion +generated_ids = model.generate( + **model_inputs, + max_new_tokens=1024 +) +output_ids = generated_ids[0][len(model_inputs.input_ids[0]):].tolist() + +# decode the generated ids +generate_text = tokenizer.decode(output_ids, skip_special_tokens=True) +print("generate_text:", generate_text) +``` + +## License + +The ERNIE 4.5 models are provided under the Apache License 2.0. This license permits commercial use, subject to its terms and conditions. Copyright (c) 2025 Baidu, Inc. All Rights Reserved. + +## Citation + +If you find ERNIE 4.5 useful or wish to use it in your projects, please kindly cite our technical report: + +```text +@misc{ernie2025technicalreport, + title={ERNIE 4.5 Technical Report}, + author={Baidu-ERNIE-Team}, + year={2025}, + primaryClass={cs.CL}, + howpublished={\url{https://ernie.baidu.com/blog/publication/ERNIE_Technical_Report.pdf}} +} +``` + diff --git a/configuration.json b/configuration.json new file mode 100644 index 0000000..bbeeda1 --- /dev/null +++ b/configuration.json @@ -0,0 +1 @@ +{"framework": "pytorch", "task": "text-generation", "allow_remote": true} \ No newline at end of file diff --git a/imatrix_unsloth.gguf_file b/imatrix_unsloth.gguf_file new file mode 100644 index 0000000..8566556 --- /dev/null +++ b/imatrix_unsloth.gguf_file @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bdd12ca1a8fd716b2b1db258ebe2ef3a050f3945a5623d92d809616dc9071859 +size 48452192