commit 04af9c7d4bea6c361a339d2d553a827a2b26780c Author: ModelHub XC Date: Sat Sep 12 19:44:17 2026 +0800 初始化项目，由ModelHub XC社区提供模型 Model: ThomasBaruzier/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-GGUF Source: Original Platform diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..72e13a8 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,64 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +imatrix.dat filter=lfs diff=lfs merge=lfs -text +DeepSeek-R1-ReDistill-Qwen-7B-v1.1-IQ1_M.gguf filter=lfs diff=lfs merge=lfs -text +DeepSeek-R1-ReDistill-Qwen-7B-v1.1-IQ1_S.gguf filter=lfs diff=lfs merge=lfs -text +DeepSeek-R1-ReDistill-Qwen-7B-v1.1-IQ2_M.gguf filter=lfs diff=lfs merge=lfs -text +DeepSeek-R1-ReDistill-Qwen-7B-v1.1-IQ2_XS.gguf filter=lfs diff=lfs merge=lfs -text +DeepSeek-R1-ReDistill-Qwen-7B-v1.1-IQ2_S.gguf filter=lfs diff=lfs merge=lfs -text +DeepSeek-R1-ReDistill-Qwen-7B-v1.1-Q2_K_S.gguf filter=lfs diff=lfs merge=lfs -text +DeepSeek-R1-ReDistill-Qwen-7B-v1.1-Q2_K.gguf filter=lfs diff=lfs merge=lfs -text +DeepSeek-R1-ReDistill-Qwen-7B-v1.1-IQ2_XXS.gguf filter=lfs diff=lfs merge=lfs -text +DeepSeek-R1-ReDistill-Qwen-7B-v1.1-IQ3_M.gguf filter=lfs diff=lfs merge=lfs -text +DeepSeek-R1-ReDistill-Qwen-7B-v1.1-IQ3_XS.gguf filter=lfs diff=lfs merge=lfs -text +DeepSeek-R1-ReDistill-Qwen-7B-v1.1-IQ3_XXS.gguf filter=lfs diff=lfs merge=lfs -text +DeepSeek-R1-ReDistill-Qwen-7B-v1.1-Q3_K_L.gguf filter=lfs diff=lfs merge=lfs -text +DeepSeek-R1-ReDistill-Qwen-7B-v1.1-Q3_K_S.gguf filter=lfs diff=lfs merge=lfs -text +DeepSeek-R1-ReDistill-Qwen-7B-v1.1-IQ3_S.gguf filter=lfs diff=lfs merge=lfs -text +DeepSeek-R1-ReDistill-Qwen-7B-v1.1-Q3_K_M.gguf filter=lfs diff=lfs merge=lfs -text +DeepSeek-R1-ReDistill-Qwen-7B-v1.1-IQ4_NL.gguf filter=lfs diff=lfs merge=lfs -text +DeepSeek-R1-ReDistill-Qwen-7B-v1.1-Q4_K_S.gguf filter=lfs diff=lfs merge=lfs -text +DeepSeek-R1-ReDistill-Qwen-7B-v1.1-Q4_K_M.gguf filter=lfs diff=lfs merge=lfs -text +DeepSeek-R1-ReDistill-Qwen-7B-v1.1-IQ4_XS.gguf filter=lfs diff=lfs merge=lfs -text +DeepSeek-R1-ReDistill-Qwen-7B-v1.1-Q5_K_M.gguf filter=lfs diff=lfs merge=lfs -text +DeepSeek-R1-ReDistill-Qwen-7B-v1.1-Q5_K_S.gguf filter=lfs diff=lfs merge=lfs -text +DeepSeek-R1-ReDistill-Qwen-7B-v1.1-Q6_K.gguf filter=lfs diff=lfs merge=lfs -text +DeepSeek-R1-ReDistill-Qwen-7B-v1.1-F16.gguf filter=lfs diff=lfs merge=lfs -text +DeepSeek-R1-ReDistill-Qwen-7B-v1.1-Q4_0.gguf filter=lfs diff=lfs merge=lfs -text +DeepSeek-R1-ReDistill-Qwen-7B-v1.1-Q4_1.gguf filter=lfs diff=lfs merge=lfs -text +DeepSeek-R1-ReDistill-Qwen-7B-v1.1-Q5_0.gguf filter=lfs diff=lfs merge=lfs -text +DeepSeek-R1-ReDistill-Qwen-7B-v1.1-Q5_1.gguf filter=lfs diff=lfs merge=lfs -text +DeepSeek-R1-ReDistill-Qwen-7B-v1.1-Q8_0.gguf filter=lfs diff=lfs merge=lfs -text diff --git a/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-F16.gguf b/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-F16.gguf new file mode 100644 index 0000000..9fd19f2 --- /dev/null +++ b/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-F16.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4e01b6c3a067974bdf6fbb5668d3ba69a0cf8f27bfd4a414a8f6e73000a206c3 +size 15237853344 diff --git a/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-IQ1_M.gguf b/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-IQ1_M.gguf new file mode 100644 index 0000000..4914b29 --- /dev/null +++ b/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-IQ1_M.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a3a169c78abcb1cc4d9a91f3dfb89142ee9b991f084e8b2b0007a0b0f240d399 +size 2042196416 diff --git a/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-IQ1_S.gguf b/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-IQ1_S.gguf new file mode 100644 index 0000000..d61e8ac --- /dev/null +++ b/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-IQ1_S.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4e4051ccd2c276ab6afac18ef0f89ca1f06470f2657555574362461312def0c5 +size 1903667648 diff --git a/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-IQ2_M.gguf b/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-IQ2_M.gguf new file mode 100644 index 0000000..63ef62c --- /dev/null +++ b/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-IQ2_M.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5e6c50cd3fef977241db6e4728a5186f2329d12cae5d7ada6c68ebce1f65c3f2 +size 2780342720 diff --git a/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-IQ2_S.gguf b/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-IQ2_S.gguf new file mode 100644 index 0000000..edf154d --- /dev/null +++ b/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-IQ2_S.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b170a212877396ad6fbe0b02edfbc6ad9871b96c3bec0f8e819a01cccb11aed0 +size 2595637696 diff --git a/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-IQ2_XS.gguf b/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-IQ2_XS.gguf new file mode 100644 index 0000000..fad551c --- /dev/null +++ b/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-IQ2_XS.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bfdcb887f45749225fd6ac77d8f76359318c1422aa678d1efe36a1b35420b167 +size 2469022144 diff --git a/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-IQ2_XXS.gguf b/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-IQ2_XXS.gguf new file mode 100644 index 0000000..03556b2 --- /dev/null +++ b/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-IQ2_XXS.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ba85e58c64554aa214e1fd990395796f707569a7896daeb9da6a51a39d6309af +size 2273077696 diff --git a/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-IQ3_M.gguf b/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-IQ3_M.gguf new file mode 100644 index 0000000..69946c7 --- /dev/null +++ b/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-IQ3_M.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c673d13f2ae0238604365efcd34dbf1adffbce2f806376e6772dc502cf31f999 +size 3574012352 diff --git a/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-IQ3_S.gguf b/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-IQ3_S.gguf new file mode 100644 index 0000000..07133f9 --- /dev/null +++ b/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-IQ3_S.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dda4d9327a8eca82a54091fc45f1a6a71d5855eddc1e3e9b7ec8855a6abdd17c +size 3499192768 diff --git a/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-IQ3_XS.gguf b/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-IQ3_XS.gguf new file mode 100644 index 0000000..b081afe --- /dev/null +++ b/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-IQ3_XS.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:db362a5614774617e1b207bf3731c7ea3c33f33d89d976883939d358e61e26d6 +size 3346256320 diff --git a/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-IQ3_XXS.gguf b/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-IQ3_XXS.gguf new file mode 100644 index 0000000..69ec15e --- /dev/null +++ b/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-IQ3_XXS.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:55860795a559e4acb88939c0babffce5f6fd2c6f2445129c4d8aa5eb5f95411e +size 3114514880 diff --git a/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-IQ4_NL.gguf b/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-IQ4_NL.gguf new file mode 100644 index 0000000..6131c3f --- /dev/null +++ b/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-IQ4_NL.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:25ad2ba3e7a16861c2b7e873b89c3e2d83fb417052a246a7158394ad55ae2464 +size 4437813696 diff --git a/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-IQ4_XS.gguf b/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-IQ4_XS.gguf new file mode 100644 index 0000000..03f3649 --- /dev/null +++ b/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-IQ4_XS.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e3540ac3a82741907a2d610134a9b76bc12beb59883e1efd9f56fb78b5ac8bc1 +size 4218472896 diff --git a/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-Q2_K.gguf b/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-Q2_K.gguf new file mode 100644 index 0000000..5e3de26 --- /dev/null +++ b/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-Q2_K.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:de9162c3859bfcaae5584cebf167de7df3fafa579abcdd01d0383358dfc4b8fe +size 3015940544 diff --git a/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-Q2_K_S.gguf b/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-Q2_K_S.gguf new file mode 100644 index 0000000..78c3507 --- /dev/null +++ b/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-Q2_K_S.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ef09e7075d3ecbcaf167bb6963179fd01705ee589824b2de150faf4b903c352f +size 2834074048 diff --git a/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-Q3_K_L.gguf b/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-Q3_K_L.gguf new file mode 100644 index 0000000..ec6ffc6 --- /dev/null +++ b/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-Q3_K_L.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:085aa0082430b579e548555c6254bc5aa34560c14ffd77f0a86895a9c884a8ac +size 4088459712 diff --git a/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-Q3_K_M.gguf b/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-Q3_K_M.gguf new file mode 100644 index 0000000..83b6471 --- /dev/null +++ b/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-Q3_K_M.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a2f1aacfd8d814a7199f1962fce9268b25b83f06c4fad34e78b2571b9def102e +size 3808391616 diff --git a/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-Q3_K_S.gguf b/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-Q3_K_S.gguf new file mode 100644 index 0000000..9b0dc11 --- /dev/null +++ b/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-Q3_K_S.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fb2a306f71acc8e9ec05e64ad97f8f943ef0e3fb8e9ba9238735b1311bbd74f6 +size 3492368832 diff --git a/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-Q4_0.gguf b/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-Q4_0.gguf new file mode 100644 index 0000000..a9f6255 --- /dev/null +++ b/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-Q4_0.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bbb724d3ca03ea5439d5d40de8f95c2a72eb0812989ae176cd249686582b808b +size 4444121536 diff --git a/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-Q4_1.gguf b/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-Q4_1.gguf new file mode 100644 index 0000000..6369b77 --- /dev/null +++ b/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-Q4_1.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e7cc848cfbdda0173b86b85cd357bbef085acba837bcf189c816d4a293966c24 +size 4873284032 diff --git a/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-Q4_K_M.gguf b/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-Q4_K_M.gguf new file mode 100644 index 0000000..25f51e0 --- /dev/null +++ b/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-Q4_K_M.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:24eca04e4225b3cde72984294c32573955dbaeef9d3ae828076cb5ba0fe09aac +size 4683073984 diff --git a/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-Q4_K_S.gguf b/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-Q4_K_S.gguf new file mode 100644 index 0000000..41eb135 --- /dev/null +++ b/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-Q4_K_S.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:01a557e5eb9d5512c8f5c1a37fb1fe709354cc1e8945cd8bbf61c98f2fc2c1f5 +size 4457769408 diff --git a/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-Q5_0.gguf b/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-Q5_0.gguf new file mode 100644 index 0000000..170d2ec --- /dev/null +++ b/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-Q5_0.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:492bcf1e03f6f6c990cae60cc7acf36444b85dad7dd2ada29d4fe97521be0b49 +size 5327907264 diff --git a/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-Q5_1.gguf b/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-Q5_1.gguf new file mode 100644 index 0000000..e3fa330 --- /dev/null +++ b/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-Q5_1.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ade55118d77312eca9a1e569c8bc5296105adc6cec777d54684cad4286be9c7a +size 5757069760 diff --git a/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-Q5_K_M.gguf b/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-Q5_K_M.gguf new file mode 100644 index 0000000..6422de6 --- /dev/null +++ b/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-Q5_K_M.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a7364685a0d6511d49b3d66f2f71267f336666e39cfba7070f1a2ed1c9ebd80a +size 5444831680 diff --git a/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-Q5_K_S.gguf b/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-Q5_K_S.gguf new file mode 100644 index 0000000..a029434 --- /dev/null +++ b/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-Q5_K_S.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0e2a7b231e603d911374fcabcc1a7ddbb221be86b4ceb436712b1473dd8867cb +size 5315176896 diff --git a/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-Q6_K.gguf b/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-Q6_K.gguf new file mode 100644 index 0000000..65a4ca5 --- /dev/null +++ b/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-Q6_K.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aec9a69e61fc102bef2ddb567eeedb32179ae34a576bf9fe246a84d1798f6a80 +size 6254199232 diff --git a/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-Q8_0.gguf b/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-Q8_0.gguf new file mode 100644 index 0000000..bd050ac --- /dev/null +++ b/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-Q8_0.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a486fe2fd1d1e23060c8145787bee7ec8a3d1f36f024e50103b2774048a81437 +size 8098525632 diff --git a/README.md b/README.md new file mode 100644 index 0000000..e30cfbf --- /dev/null +++ b/README.md @@ -0,0 +1,175 @@ +--- +license: mit +train: false +inference: true +pipeline_tag: text-generation +base_model: +- deepseek-ai/DeepSeek-R1-Distill-Qwen-7B +--- + +

+ +# Llama.cpp imatrix quantizations of [mobiuslabsgmbh/DeepSeek-R1-ReDistill-Qwen-7B-v1.1](https://huggingface.co/mobiuslabsgmbh/DeepSeek-R1-ReDistill-Qwen-7B-v1.1) + +Using llama.cpp commit [3ad5451](https://github.com/ggerganov/llama.cpp/commit/3ad5451) for quantization. + +All quants were made using the imatrix option and Bartowski's [calibration file](https://gist.github.com/bartowski1182/eb213dccb3571f863da82e99418f81e8). + +
+ +# Perplexity table (the lower the better) + +| Quant | Size (MB) | PPL | Size (%) | Accuracy (%) | PPL error rate | +| -------------------------------------------------------------------------------------------------------------------------------------------------- | --------- | ------- | -------- | ------------ | -------------- | +| [IQ1_S](https://huggingface.co/ThomasBaruzier/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-GGUF/blob/main/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-IQ1_S.gguf) | 1815 | 29.3739 | 12.49 | 49.92 | 0.53 | +| [IQ1_M](https://huggingface.co/ThomasBaruzier/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-GGUF/blob/main/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-IQ1_M.gguf) | 1947 | 23.4611 | 13.40 | 62.50 | 0.42 | +| [IQ2_XXS](https://huggingface.co/ThomasBaruzier/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-GGUF/blob/main/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-IQ2_XXS.gguf) | 2167 | 23.8257 | 14.91 | 61.54 | 0.46 | +| [IQ2_XS](https://huggingface.co/ThomasBaruzier/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-GGUF/blob/main/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-IQ2_XS.gguf) | 2354 | 20.5413 | 16.20 | 71.38 | 0.39 | +| [IQ2_S](https://huggingface.co/ThomasBaruzier/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-GGUF/blob/main/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-IQ2_S.gguf) | 2475 | 19.3763 | 17.03 | 75.67 | 0.36 | +| [IQ2_M](https://huggingface.co/ThomasBaruzier/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-GGUF/blob/main/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-IQ2_M.gguf) | 2651 | 22.3007 | 18.24 | 65.75 | 0.44 | +| [Q2_K_S](https://huggingface.co/ThomasBaruzier/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-GGUF/blob/main/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-Q2_K_S.gguf) | 2702 | 17.5446 | 18.59 | 83.57 | 0.31 | +| [Q2_K](https://huggingface.co/ThomasBaruzier/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-GGUF/blob/main/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-Q2_K.gguf) | 2876 | 16.9426 | 19.79 | 86.54 | 0.29 | +| [IQ3_XXS](https://huggingface.co/ThomasBaruzier/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-GGUF/blob/main/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-IQ3_XXS.gguf) | 2970 | 16.2668 | 20.44 | 90.14 | 0.29 | +| [IQ3_XS](https://huggingface.co/ThomasBaruzier/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-GGUF/blob/main/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-IQ3_XS.gguf) | 3191 | 16.1443 | 21.96 | 90.82 | 0.29 | +| [Q3_K_S](https://huggingface.co/ThomasBaruzier/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-GGUF/blob/main/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-Q3_K_S.gguf) | 3330 | 17.0364 | 22.92 | 86.07 | 0.29 | +| [IQ3_S](https://huggingface.co/ThomasBaruzier/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-GGUF/blob/main/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-IQ3_S.gguf) | 3337 | 16.1048 | 22.96 | 91.04 | 0.29 | +| [IQ3_M](https://huggingface.co/ThomasBaruzier/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-GGUF/blob/main/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-IQ3_M.gguf) | 3408 | 15.8128 | 23.45 | 92.72 | 0.28 | +| [Q3_K_M](https://huggingface.co/ThomasBaruzier/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-GGUF/blob/main/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-Q3_K_M.gguf) | 3631 | 15.2580 | 24.99 | 96.10 | 0.26 | +| [Q3_K_L](https://huggingface.co/ThomasBaruzier/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-GGUF/blob/main/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-Q3_K_L.gguf) | 3899 | 15.1997 | 26.83 | 96.46 | 0.26 | +| [IQ4_XS](https://huggingface.co/ThomasBaruzier/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-GGUF/blob/main/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-IQ4_XS.gguf) | 4023 | 14.9385 | 27.68 | 98.15 | 0.25 | +| [IQ4_NL](https://huggingface.co/ThomasBaruzier/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-GGUF/blob/main/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-IQ4_NL.gguf) | 4232 | 14.9257 | 29.12 | 98.24 | 0.25 | +| [Q4_0](https://huggingface.co/ThomasBaruzier/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-GGUF/blob/main/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-Q4_0.gguf) | 4238 | 15.2621 | 29.17 | 96.07 | 0.26 | +| [Q4_K_S](https://huggingface.co/ThomasBaruzier/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-GGUF/blob/main/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-Q4_K_S.gguf) | 4251 | 14.8852 | 29.25 | 98.50 | 0.26 | +| [Q4_K_M](https://huggingface.co/ThomasBaruzier/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-GGUF/blob/main/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-Q4_K_M.gguf) | 4466 | 14.8666 | 30.73 | 98.63 | 0.26 | +| [Q4_1](https://huggingface.co/ThomasBaruzier/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-GGUF/blob/main/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-Q4_1.gguf) | 4647 | 14.8789 | 31.98 | 98.54 | 0.26 | +| [Q5_K_S](https://huggingface.co/ThomasBaruzier/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-GGUF/blob/main/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-Q5_K_S.gguf) | 5068 | 14.7449 | 34.88 | 99.44 | 0.25 | +| [Q5_0](https://huggingface.co/ThomasBaruzier/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-GGUF/blob/main/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-Q5_0.gguf) | 5081 | 14.7425 | 34.97 | 99.46 | 0.25 | +| [Q5_K_M](https://huggingface.co/ThomasBaruzier/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-GGUF/blob/main/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-Q5_K_M.gguf) | 5192 | 14.7327 | 35.73 | 99.52 | 0.25 | +| [Q5_1](https://huggingface.co/ThomasBaruzier/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-GGUF/blob/main/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-Q5_1.gguf) | 5490 | 14.7293 | 37.78 | 99.55 | 0.25 | +| [Q6_K](https://huggingface.co/ThomasBaruzier/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-GGUF/blob/main/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-Q6_K.gguf) | 5964 | 14.6907 | 41.04 | 99.81 | 0.25 | +| [Q8_0](https://huggingface.co/ThomasBaruzier/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-GGUF/blob/main/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-Q8_0.gguf) | 7723 | 14.6686 | 53.15 | 99.96 | 0.25 | +| [F16](https://huggingface.co/ThomasBaruzier/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-GGUF/blob/main/DeepSeek-R1-ReDistill-Qwen-7B-v1.1-F16.gguf) | 14531 | 14.6625 | 100 | 100 | 0.25 | + +
+ +This is a version of the DeepSeek-R1-Distill-Qwen-7B model re-distilled for better performance. + +## Performance + +| Models | DeepSeek-R1-Distill-Qwen-7B | DeepSeek-R1-ReDistill-Qwen-7B-v1.1 | +|:-------------------:|:--------:|:----------------:| +| ARC (25-shot) | 55.03 | 52.3 | +| HellaSwag (10-shot)| 61.9 | 62.36 | +| MMLU (5-shot) | 56.75 | 59.53 | +| TruthfulQA-MC2 | 45.76 | 47.7 | +| Winogrande (5-shot)| 60.38 | 61.8 | +| GSM8K (5-shot) | 78.85 | 83.4 | +| Average | 59.78 | 61.18 | + +| Models | DeepSeek-R1-Distill-Qwen-7B | DeepSeek-R1-ReDistill-Qwen-7B-v1.1 | +|:-------------------:|:--------:|:----------------:| +| GPQA (0-shot) | 30.9 | 34.99 | +| MMLU PRO (5-shot) | 28.83 | 31.02 | +| MUSR (0-shot) | 38.85 | 44.42 | +| BBH (3-shot) | 43.54 | 51.53 | +| IfEval (0-shot) - strict | 42.33 | 35.49 | +| IfEval (0-shot) - loose | 30.31 | 38.49 | + +## Usage +```Python +import torch +from transformers import AutoModelForCausalLM, AutoTokenizer +compute_dtype = torch.bfloat16 +device = 'cuda' +model_id = "mobiuslabsgmbh/DeepSeek-R1-ReDistill-Qwen-7B-v1.1" + +model = AutoModelForCausalLM.from_pretrained(model_id, torch_dtype=compute_dtype, attn_implementation="sdpa", device_map=device) +tokenizer = AutoTokenizer.from_pretrained(model_id) + +prompt = "What is 1.5+102.2?" +chat = tokenizer.apply_chat_template([{"role":"user", "content":prompt}], tokenize=True, add_generation_prompt=True, return_tensors="pt") +outputs = model.generate(chat.to(device), max_new_tokens=1024, do_sample=True) +print(tokenizer.decode(outputs[0])) +``` + +Output: +``` +<｜begin▁of▁sentence｜><｜User｜>What is 1.5+102.2?<｜Assistant｜> +First, I need to add the whole number parts of the two numbers. The whole numbers are 1 and 102, which add up to 103. + +Next, I add the decimal parts of the two numbers. The decimal parts are 0.5 and 0.2, which add up to 0.7. + +Finally, I combine the whole number and decimal parts to get the total sum. Adding 103 and 0.7 gives me 103.7. + + +To add the numbers \(1.5\) and \(102.2\), follow these steps: + +1. **Add the whole number parts:** + \[ + 1 + 102 = 103 + \] + +2. **Add the decimal parts:** + \[ + 0.5 + 0.2 = 0.7 + \] + +3. **Combine the results:** + \[ + 103 + 0.7 = 103.7 + \] + +**Final Answer:** +\[ +\boxed{103.7} +\]<｜end▁of▁sentence｜> +``` + +## HQQ +Run ~3.5x faster with HQQ. First, install the dependencies: +``` +pip install hqq +``` + +```Python +import torch +from transformers import AutoModelForCausalLM, AutoTokenizer +from hqq.models.hf.base import AutoHQQHFModel +from hqq.core.quantize import * + +#Params +device = 'cuda:0' +backend = "torchao_int4" +compute_dtype = torch.bfloat16 if backend=="torchao_int4" else torch.float16 +model_id = "mobiuslabsgmbh/DeepSeek-R1-ReDistill-Qwen-7B-v1.1" + +#Load +tokenizer = AutoTokenizer.from_pretrained(model_id) +model = AutoModelForCausalLM.from_pretrained(model_id, torch_dtype=compute_dtype, attn_implementation="sdpa") + +#Quantize +quant_config = BaseQuantizeConfig(nbits=4, group_size=64, axis=1) +AutoHQQHFModel.quantize_model(model, quant_config=quant_config, compute_dtype=compute_dtype, device=device) + +#Optimize +from hqq.utils.patching import prepare_for_inference +prepare_for_inference(model, backend=backend, verbose=False) + +############################################################ +#Generate (streaming) +from hqq.utils.generation_hf import HFGenerator +gen = HFGenerator(model, tokenizer, max_new_tokens=4096, do_sample=True, compile='partial').warmup() + +prompt = "If A equals B, and C equals B - A, what would be the value of C?" +out = gen.generate(prompt, print_tokens=True) + +############################################################ +# #Generate (simple) +# from hqq.utils.generation_hf import patch_model_for_compiled_runtime +# patch_model_for_compiled_runtime(model, tokenizer, warmup=True) + +# prompt = "If A equals B, and C equals B - A, what would be the value of C?" +# chat = tokenizer.apply_chat_template([{"role":"user", "content":prompt}], tokenize=True, add_generation_prompt=True, return_tensors="pt") +# outputs = model.generate(chat.to(device), max_new_tokens=8192, do_sample=True) +# print(tokenizer.decode(outputs[0])) +``` \ No newline at end of file diff --git a/imatrix.dat b/imatrix.dat new file mode 100644 index 0000000..7520405 --- /dev/null +++ b/imatrix.dat @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:190732cb7fbb24ebbfee9c2d8fc1493afcfae8b7b1b48e72abe6b8555d8d0d56 +size 4536696 diff --git a/perplexity.md b/perplexity.md new file mode 100644 index 0000000..4b982ef --- /dev/null +++ b/perplexity.md @@ -0,0 +1,30 @@ +| Quant | Size (MB) | PPL | Size (%) | Accuracy (%) | PPL error rate | +| ------ | --------- | ------- | -------- | ------------ | -------------- | +| IQ1_S | 1815 | 29.3739 | 12.49 | 49.92 | 0.53 | +| IQ1_M | 1947 | 23.4611 | 13.40 | 62.50 | 0.42 | +| IQ2_XXS | 2167 | 23.8257 | 14.91 | 61.54 | 0.46 | +| IQ2_XS | 2354 | 20.5413 | 16.20 | 71.38 | 0.39 | +| IQ2_S | 2475 | 19.3763 | 17.03 | 75.67 | 0.36 | +| IQ2_M | 2651 | 22.3007 | 18.24 | 65.75 | 0.44 | +| Q2_K_S | 2702 | 17.5446 | 18.59 | 83.57 | 0.31 | +| Q2_K | 2876 | 16.9426 | 19.79 | 86.54 | 0.29 | +| IQ3_XXS | 2970 | 16.2668 | 20.44 | 90.14 | 0.29 | +| IQ3_XS | 3191 | 16.1443 | 21.96 | 90.82 | 0.29 | +| Q3_K_S | 3330 | 17.0364 | 22.92 | 86.07 | 0.29 | +| IQ3_S | 3337 | 16.1048 | 22.96 | 91.04 | 0.29 | +| IQ3_M | 3408 | 15.8128 | 23.45 | 92.72 | 0.28 | +| Q3_K_M | 3631 | 15.2580 | 24.99 | 96.10 | 0.26 | +| Q3_K_L | 3899 | 15.1997 | 26.83 | 96.46 | 0.26 | +| IQ4_XS | 4023 | 14.9385 | 27.68 | 98.15 | 0.25 | +| IQ4_NL | 4232 | 14.9257 | 29.12 | 98.24 | 0.25 | +| Q4_0 | 4238 | 15.2621 | 29.17 | 96.07 | 0.26 | +| Q4_K_S | 4251 | 14.8852 | 29.25 | 98.50 | 0.26 | +| Q4_K_M | 4466 | 14.8666 | 30.73 | 98.63 | 0.26 | +| Q4_1 | 4647 | 14.8789 | 31.98 | 98.54 | 0.26 | +| Q5_K_S | 5068 | 14.7449 | 34.88 | 99.44 | 0.25 | +| Q5_0 | 5081 | 14.7425 | 34.97 | 99.46 | 0.25 | +| Q5_K_M | 5192 | 14.7327 | 35.73 | 99.52 | 0.25 | +| Q5_1 | 5490 | 14.7293 | 37.78 | 99.55 | 0.25 | +| Q6_K | 5964 | 14.6907 | 41.04 | 99.81 | 0.25 | +| Q8_0 | 7723 | 14.6686 | 53.15 | 99.96 | 0.25 | +| F16 | 14531 | 14.6625 | 100 | 100 | 0.25 |