From 9f701934a2b5e311f98a77c5aa471cda10562aff Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Tue, 14 Jul 2026 18:50:12 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: h2oai/h2o-danube3-4b-chat-GGUF Source: Original Platform --- .gitattributes | 43 ++++++++++++++++++++++++++++ README.md | 50 +++++++++++++++++++++++++++++++++ h2o-danube3-4b-chat-BF16.gguf | 3 ++ h2o-danube3-4b-chat-F16.gguf | 3 ++ h2o-danube3-4b-chat-Q2_K.gguf | 3 ++ h2o-danube3-4b-chat-Q3_K_M.gguf | 3 ++ h2o-danube3-4b-chat-Q4_K_M.gguf | 3 ++ h2o-danube3-4b-chat-Q5_K_M.gguf | 3 ++ h2o-danube3-4b-chat-Q6_K.gguf | 3 ++ h2o-danube3-4b-chat-Q8_0.gguf | 3 ++ 10 files changed, 117 insertions(+) create mode 100644 .gitattributes create mode 100644 README.md create mode 100644 h2o-danube3-4b-chat-BF16.gguf create mode 100644 h2o-danube3-4b-chat-F16.gguf create mode 100644 h2o-danube3-4b-chat-Q2_K.gguf create mode 100644 h2o-danube3-4b-chat-Q3_K_M.gguf create mode 100644 h2o-danube3-4b-chat-Q4_K_M.gguf create mode 100644 h2o-danube3-4b-chat-Q5_K_M.gguf create mode 100644 h2o-danube3-4b-chat-Q6_K.gguf create mode 100644 h2o-danube3-4b-chat-Q8_0.gguf diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..2288111 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,43 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +h2o-danube3-4b-chat-BF16.gguf filter=lfs diff=lfs merge=lfs -text +h2o-danube3-4b-chat-F16.gguf filter=lfs diff=lfs merge=lfs -text +h2o-danube3-4b-chat-Q2_K.gguf filter=lfs diff=lfs merge=lfs -text +h2o-danube3-4b-chat-Q3_K_M.gguf filter=lfs diff=lfs merge=lfs -text +h2o-danube3-4b-chat-Q4_K_M.gguf filter=lfs diff=lfs merge=lfs -text +h2o-danube3-4b-chat-Q5_K_M.gguf filter=lfs diff=lfs merge=lfs -text +h2o-danube3-4b-chat-Q6_K.gguf filter=lfs diff=lfs merge=lfs -text +h2o-danube3-4b-chat-Q8_0.gguf filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..a3ec705 --- /dev/null +++ b/README.md @@ -0,0 +1,50 @@ +--- +language: +- en +library_name: transformers +license: apache-2.0 +tags: +- gpt +- llm +- large language model +- h2o-llmstudio +thumbnail: >- + https://h2o.ai/etc.clientlibs/h2o/clientlibs/clientlib-site/resources/images/favicon.ico +pipeline_tag: text-generation +quantized_by: h2oai +--- + +# h2o-danube3-4b-chat-GGUF +- Model creator: [H2O.ai](https://huggingface.co/h2oai) +- Original model: [h2oai/h2o-danube3-4b-chat](https://huggingface.co/h2oai/h2o-danube3-4b-chat) + +## Description + +This repo contains GGUF format model files for [h2o-danube3-4b-chat](https://huggingface.co/h2oai/h2o-danube3-4b-chat) quantized using [llama.cpp](https://github.com/ggerganov/llama.cpp/) framework. + +Table below summarizes different quantized versions of [h2o-danube3-4b-chat](https://huggingface.co/h2oai/h2o-danube3-4b-chat). It shows the trade-off between size, speed and quality of the models. + + +| Name | Quant method | Model size | MT-Bench AVG | Perplexity | Tokens per second | +|:----------------------------------|:----------------------------------:|:----------:|:------------:|:------------:|:-------------------:| +| [h2o-danube3-4b-chat-F16.gguf](https://huggingface.co/h2oai/h2o-danube3-4b-chat-GGUF/blob/main/h2o-danube3-4b-chat-F16.gguf) | F16 | 7.92 GB | 6.43 | 6.17 | 479 | +| [h2o-danube3-4b-chat-Q8_0.gguf](https://huggingface.co/h2oai/h2o-danube3-4b-chat-GGUF/blob/main/h2o-danube3-4b-chat-Q8_0.gguf) | Q8_0 | 4.21 GB | 6.49 | 6.17 | 725 | +| [h2o-danube3-4b-chat-Q6_K.gguf](https://huggingface.co/h2oai/h2o-danube3-4b-chat-GGUF/blob/main/h2o-danube3-4b-chat-Q6_K.gguf) | Q6_K | 3.25 GB | 6.37 | 6.20 | 791 | +| [h2o-danube3-4b-chat-Q5_K_M.gguf](https://huggingface.co/h2oai/h2o-danube3-4b-chat-GGUF/blob/main/h2o-danube3-4b-chat-Q5_K_M.gguf) | Q5_K_M | 2.81 GB | 6.25 | 6.24 | 927 | +| [h2o-danube3-4b-chat-Q4_K_M.gguf](https://huggingface.co/h2oai/h2o-danube3-4b-chat-GGUF/blob/main/h2o-danube3-4b-chat-Q4_K_M.gguf) | Q4_K_M | 2.39 GB | 6.31 | 6.37 | 967 | +| [h2o-danube3-4b-chat-Q3_K_M.gguf](https://huggingface.co/h2oai/h2o-danube3-4b-chat-GGUF/blob/main/h2o-danube3-4b-chat-Q3_K_M.gguf) | Q3_K_M | 1.94 GB | 5.87 | 6.99 | 1099 | +| [h2o-danube3-4b-chat-Q2_K.gguf](https://huggingface.co/h2oai/h2o-danube3-4b-chat-GGUF/blob/main/h2o-danube3-4b-chat-Q2_K.gguf) | Q2_K | 1.51 GB | 3.71 | 9.42 | 1299 | + +Columns in the table are: +* Name -- model name and link +* Quant method -- quantization method +* Model size -- size of the model in gigabytes +* MT-Bench AVG -- [MT-Bench](https://arxiv.org/abs/2306.05685) benchmark score. The score is from 1 to 10, the higher, the better +* Perplexity -- perplexity metric on WikiText-2 dataset. It's reported in a perplexity test from llama.cpp. The lower, the better +* Tokens per second -- generation speed in tokens per second, as reported in a perplexity test from llama.cpp. The higher, the better. Speed tests are done on a single H100 GPU + + +## Prompt template +``` +<|prompt|>Why is drinking water so healthy?<|answer|> +``` diff --git a/h2o-danube3-4b-chat-BF16.gguf b/h2o-danube3-4b-chat-BF16.gguf new file mode 100644 index 0000000..5374f15 --- /dev/null +++ b/h2o-danube3-4b-chat-BF16.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b99ca5f2c693b19aaf930bc38a7d0334a71617abedb9cba7455b6e75183bb72e +size 7924786560 diff --git a/h2o-danube3-4b-chat-F16.gguf b/h2o-danube3-4b-chat-F16.gguf new file mode 100644 index 0000000..37b75a4 --- /dev/null +++ b/h2o-danube3-4b-chat-F16.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b82e4db1c964babd272acf005c8e53e01d8e256dc59c2622807f734de0d8d31a +size 7924786560 diff --git a/h2o-danube3-4b-chat-Q2_K.gguf b/h2o-danube3-4b-chat-Q2_K.gguf new file mode 100644 index 0000000..6fddf25 --- /dev/null +++ b/h2o-danube3-4b-chat-Q2_K.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b714e5a1f77c8edc18631d43029fddf528a94004bc8df44843e6de1cff38f0aa +size 1514405760 diff --git a/h2o-danube3-4b-chat-Q3_K_M.gguf b/h2o-danube3-4b-chat-Q3_K_M.gguf new file mode 100644 index 0000000..9dcc2db --- /dev/null +++ b/h2o-danube3-4b-chat-Q3_K_M.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7cea68848961fcac1d8793d2065d097e18480f87fe2f303aa29833456eb588e5 +size 1941682560 diff --git a/h2o-danube3-4b-chat-Q4_K_M.gguf b/h2o-danube3-4b-chat-Q4_K_M.gguf new file mode 100644 index 0000000..f829ffe --- /dev/null +++ b/h2o-danube3-4b-chat-Q4_K_M.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:74abba3fd4abed59f35c3b093a08d7705039976f6ace4b6428071acf2c4ccd53 +size 2394648960 diff --git a/h2o-danube3-4b-chat-Q5_K_M.gguf b/h2o-danube3-4b-chat-Q5_K_M.gguf new file mode 100644 index 0000000..a8c9276 --- /dev/null +++ b/h2o-danube3-4b-chat-Q5_K_M.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c3f932f009d09facf797e0f126bb0bc36a081d663165bc1f3fe9609b3d55c1f8 +size 2809983360 diff --git a/h2o-danube3-4b-chat-Q6_K.gguf b/h2o-danube3-4b-chat-Q6_K.gguf new file mode 100644 index 0000000..b8627bb --- /dev/null +++ b/h2o-danube3-4b-chat-Q6_K.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f6d27d857e786541196b51901019ed4b32ebb639d8eb6b3f16cd2da417f0e06 +size 3251276160 diff --git a/h2o-danube3-4b-chat-Q8_0.gguf b/h2o-danube3-4b-chat-Q8_0.gguf new file mode 100644 index 0000000..48f0d7e --- /dev/null +++ b/h2o-danube3-4b-chat-Q8_0.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:913b65eb8d7afbb2e62bd345448e9b60c2ab09da5066e503e6e74ef875d536fa +size 4210738560