From ec3471c6d22b316055345e347ae837b2f7674db6 Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Mon, 12 Oct 2026 05:11:17 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: aashish1904/Jan-v3-4B-base-instruct-GGUF Source: Original Platform --- .gitattributes | 49 ++++++++++ Jan-v3-4B-base-instruct.Q2_K.gguf | 3 + Jan-v3-4B-base-instruct.Q3_K_L.gguf | 3 + Jan-v3-4B-base-instruct.Q3_K_M.gguf | 3 + Jan-v3-4B-base-instruct.Q3_K_S.gguf | 3 + Jan-v3-4B-base-instruct.Q4_0.gguf | 3 + Jan-v3-4B-base-instruct.Q4_1.gguf | 3 + Jan-v3-4B-base-instruct.Q4_K_M.gguf | 3 + Jan-v3-4B-base-instruct.Q4_K_S.gguf | 3 + Jan-v3-4B-base-instruct.Q5_0.gguf | 3 + Jan-v3-4B-base-instruct.Q5_1.gguf | 3 + Jan-v3-4B-base-instruct.Q5_K_M.gguf | 3 + Jan-v3-4B-base-instruct.Q5_K_S.gguf | 3 + Jan-v3-4B-base-instruct.Q6_K.gguf | 3 + Jan-v3-4B-base-instruct.Q8_0.gguf | 3 + README.md | 134 ++++++++++++++++++++++++++++ 16 files changed, 225 insertions(+) create mode 100644 .gitattributes create mode 100644 Jan-v3-4B-base-instruct.Q2_K.gguf create mode 100644 Jan-v3-4B-base-instruct.Q3_K_L.gguf create mode 100644 Jan-v3-4B-base-instruct.Q3_K_M.gguf create mode 100644 Jan-v3-4B-base-instruct.Q3_K_S.gguf create mode 100644 Jan-v3-4B-base-instruct.Q4_0.gguf create mode 100644 Jan-v3-4B-base-instruct.Q4_1.gguf create mode 100644 Jan-v3-4B-base-instruct.Q4_K_M.gguf create mode 100644 Jan-v3-4B-base-instruct.Q4_K_S.gguf create mode 100644 Jan-v3-4B-base-instruct.Q5_0.gguf create mode 100644 Jan-v3-4B-base-instruct.Q5_1.gguf create mode 100644 Jan-v3-4B-base-instruct.Q5_K_M.gguf create mode 100644 Jan-v3-4B-base-instruct.Q5_K_S.gguf create mode 100644 Jan-v3-4B-base-instruct.Q6_K.gguf create mode 100644 Jan-v3-4B-base-instruct.Q8_0.gguf create mode 100644 README.md diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..e6faeff --- /dev/null +++ b/.gitattributes @@ -0,0 +1,49 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +Jan-v3-4B-base-instruct.Q2_K.gguf filter=lfs diff=lfs merge=lfs -text +Jan-v3-4B-base-instruct.Q3_K_S.gguf filter=lfs diff=lfs merge=lfs -text +Jan-v3-4B-base-instruct.Q3_K_M.gguf filter=lfs diff=lfs merge=lfs -text +Jan-v3-4B-base-instruct.Q3_K_L.gguf filter=lfs diff=lfs merge=lfs -text +Jan-v3-4B-base-instruct.Q4_0.gguf filter=lfs diff=lfs merge=lfs -text +Jan-v3-4B-base-instruct.Q4_1.gguf filter=lfs diff=lfs merge=lfs -text +Jan-v3-4B-base-instruct.Q4_K_S.gguf filter=lfs diff=lfs merge=lfs -text +Jan-v3-4B-base-instruct.Q4_K_M.gguf filter=lfs diff=lfs merge=lfs -text +Jan-v3-4B-base-instruct.Q5_0.gguf filter=lfs diff=lfs merge=lfs -text +Jan-v3-4B-base-instruct.Q5_1.gguf filter=lfs diff=lfs merge=lfs -text +Jan-v3-4B-base-instruct.Q5_K_S.gguf filter=lfs diff=lfs merge=lfs -text +Jan-v3-4B-base-instruct.Q5_K_M.gguf filter=lfs diff=lfs merge=lfs -text +Jan-v3-4B-base-instruct.Q6_K.gguf filter=lfs diff=lfs merge=lfs -text +Jan-v3-4B-base-instruct.Q8_0.gguf filter=lfs diff=lfs merge=lfs -text diff --git a/Jan-v3-4B-base-instruct.Q2_K.gguf b/Jan-v3-4B-base-instruct.Q2_K.gguf new file mode 100644 index 0000000..d272b4a --- /dev/null +++ b/Jan-v3-4B-base-instruct.Q2_K.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d7e89e30869dc3fcec3c292547b9ad56a5cdfed65de7a6c7df35a8df901b0283 +size 1797124512 diff --git a/Jan-v3-4B-base-instruct.Q3_K_L.gguf b/Jan-v3-4B-base-instruct.Q3_K_L.gguf new file mode 100644 index 0000000..706f368 --- /dev/null +++ b/Jan-v3-4B-base-instruct.Q3_K_L.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ceef3fee5c4d0eefb2bf97e1bcb930b4efe3353cba54e9b0dad4e01d11f9ae4 +size 2406913952 diff --git a/Jan-v3-4B-base-instruct.Q3_K_M.gguf b/Jan-v3-4B-base-instruct.Q3_K_M.gguf new file mode 100644 index 0000000..fab3dea --- /dev/null +++ b/Jan-v3-4B-base-instruct.Q3_K_M.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:23439144a3f3692a9d2fafd409c5a7facc8248f6322ccf1ab3c4b48fbf469e30 +size 2242746272 diff --git a/Jan-v3-4B-base-instruct.Q3_K_S.gguf b/Jan-v3-4B-base-instruct.Q3_K_S.gguf new file mode 100644 index 0000000..41f98fd --- /dev/null +++ b/Jan-v3-4B-base-instruct.Q3_K_S.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3939d2182b61c1bb078a51ab8e8e8f15e5416917c23dcc49efc7c84397ef504c +size 2054125472 diff --git a/Jan-v3-4B-base-instruct.Q4_0.gguf b/Jan-v3-4B-base-instruct.Q4_0.gguf new file mode 100644 index 0000000..77b0679 --- /dev/null +++ b/Jan-v3-4B-base-instruct.Q4_0.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6376f7bccbdd13777ef1523abe7f5b87509149619935bebbbf9d50f458fc74b1 +size 2588333472 diff --git a/Jan-v3-4B-base-instruct.Q4_1.gguf b/Jan-v3-4B-base-instruct.Q4_1.gguf new file mode 100644 index 0000000..a532085 --- /dev/null +++ b/Jan-v3-4B-base-instruct.Q4_1.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:de04e56cbb191a695696d45e34717660e643fa58b7414213ad03c173e293ee7c +size 2839725472 diff --git a/Jan-v3-4B-base-instruct.Q4_K_M.gguf b/Jan-v3-4B-base-instruct.Q4_K_M.gguf new file mode 100644 index 0000000..e6144b7 --- /dev/null +++ b/Jan-v3-4B-base-instruct.Q4_K_M.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:17f17b38e58c46629217fd8d99571b7ff8ea3f27054b5a5923a0059af613498a +size 2716067232 diff --git a/Jan-v3-4B-base-instruct.Q4_K_S.gguf b/Jan-v3-4B-base-instruct.Q4_K_S.gguf new file mode 100644 index 0000000..bb0d3bf --- /dev/null +++ b/Jan-v3-4B-base-instruct.Q4_K_S.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7b9d86dfe1930cf248e624a7fe3964322c623a24340dcdf98d7392112fb88521 +size 2602096032 diff --git a/Jan-v3-4B-base-instruct.Q5_0.gguf b/Jan-v3-4B-base-instruct.Q5_0.gguf new file mode 100644 index 0000000..56c989a --- /dev/null +++ b/Jan-v3-4B-base-instruct.Q5_0.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1590b0973f304317b2c192db4a793c4c1f390ab2071a25b5f3af69e9afff50f9 +size 3091117472 diff --git a/Jan-v3-4B-base-instruct.Q5_1.gguf b/Jan-v3-4B-base-instruct.Q5_1.gguf new file mode 100644 index 0000000..1660e7b --- /dev/null +++ b/Jan-v3-4B-base-instruct.Q5_1.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ff086a8774fd2c8fdbcaea6d953878b330c2fdb2bf85d2d79f7978cbb865e53a +size 3342509472 diff --git a/Jan-v3-4B-base-instruct.Q5_K_M.gguf b/Jan-v3-4B-base-instruct.Q5_K_M.gguf new file mode 100644 index 0000000..48fcaa5 --- /dev/null +++ b/Jan-v3-4B-base-instruct.Q5_K_M.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:db2aa5329673cee642909a545aa29f1c2ce04a39cfcf30fe899c5f7e447dc235 +size 3156919712 diff --git a/Jan-v3-4B-base-instruct.Q5_K_S.gguf b/Jan-v3-4B-base-instruct.Q5_K_S.gguf new file mode 100644 index 0000000..c63afd3 --- /dev/null +++ b/Jan-v3-4B-base-instruct.Q5_K_S.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:11d23a41301ba461584a5f26b9b47b80dd661f91899a47679631e3dd52e52c2a +size 3091117472 diff --git a/Jan-v3-4B-base-instruct.Q6_K.gguf b/Jan-v3-4B-base-instruct.Q6_K.gguf new file mode 100644 index 0000000..a92d8ad --- /dev/null +++ b/Jan-v3-4B-base-instruct.Q6_K.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4ea80da18002d6048ddbc67796d4e21c9f0ec36401f6ad284188dde28b1f73b7 +size 3625325472 diff --git a/Jan-v3-4B-base-instruct.Q8_0.gguf b/Jan-v3-4B-base-instruct.Q8_0.gguf new file mode 100644 index 0000000..47f758a --- /dev/null +++ b/Jan-v3-4B-base-instruct.Q8_0.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5da07497c5d14dc98403b97527dab87733a010a0fe248a54f8f71e1378f5800 +size 4693669792 diff --git a/README.md b/README.md new file mode 100644 index 0000000..5ddb8e2 --- /dev/null +++ b/README.md @@ -0,0 +1,134 @@ +--- +base_model: janhq/Jan-v3-4B-base-instruct +library_name: gguf +pipeline_tag: text-generation +tags: + - gguf + - quantized + - llama-cpp +--- + +# Jan-v3-4B-base-instruct - GGUF + +This is a quantized GGUF version of [janhq/Jan-v3-4B-base-instruct](https://huggingface.co/janhq/Jan-v3-4B-base-instruct) created using [llama.cpp](https://github.com/ggerganov/llama.cpp). + +## Available Quantizations + +| Filename | Quant Type | Description | +|----------|------------|-------------| +| Jan-v3-4B-base-instruct.Q2_K.gguf | Q2_K | Smallest, significant quality loss | +| Jan-v3-4B-base-instruct.Q3_K_S.gguf | Q3_K_S | Very small, low quality | +| Jan-v3-4B-base-instruct.Q3_K_M.gguf | Q3_K_M | Very small, medium quality | +| Jan-v3-4B-base-instruct.Q3_K_L.gguf | Q3_K_L | Small, better quality than Q3_K_M | +| Jan-v3-4B-base-instruct.Q4_0.gguf | Q4_0 | Small, legacy format | +| Jan-v3-4B-base-instruct.Q4_1.gguf | Q4_1 | Small, legacy format with better accuracy | +| Jan-v3-4B-base-instruct.Q4_K_S.gguf | Q4_K_S | Small, good quality | +| Jan-v3-4B-base-instruct.Q4_K_M.gguf | Q4_K_M | Medium, balanced quality - recommended | +| Jan-v3-4B-base-instruct.Q5_0.gguf | Q5_0 | Medium, legacy format | +| Jan-v3-4B-base-instruct.Q5_1.gguf | Q5_1 | Medium, legacy format with better accuracy | +| Jan-v3-4B-base-instruct.Q5_K_S.gguf | Q5_K_S | Medium, good quality | +| Jan-v3-4B-base-instruct.Q5_K_M.gguf | Q5_K_M | Medium, high quality - recommended | +| Jan-v3-4B-base-instruct.Q6_K.gguf | Q6_K | Large, very high quality | +| Jan-v3-4B-base-instruct.Q8_0.gguf | Q8_0 | Large, near-lossless quality | + + +## Usage + +### With llama.cpp + +```bash +./llama-cli -m Jan-v3-4B-base-instruct.Q4_K_M.gguf -p "Your prompt here" +``` + +### With Ollama + +```bash +ollama run hf.co/aashish1904/Jan-v3-4B-base-instruct-GGUF +``` + +## Original Model + +- **Source**: [janhq/Jan-v3-4B-base-instruct](https://huggingface.co/janhq/Jan-v3-4B-base-instruct) +- **Quantized by**: GGUF Quantizer Space + +--- + +## Original Model Card + +# Jan-v3-4B-base-instruct: a 4B baseline model for fine-tuning + +[![GitHub](https://img.shields.io/badge/GitHub-Repository-blue?logo=github)](https://github.com/janhq/jan) +[![License](https://img.shields.io/badge/License-Apache%202.0-yellow)](https://opensource.org/licenses/Apache-2.0) +[![Jan App](https://img.shields.io/badge/Powered%20by-Jan%20App-purple?style=flat&logo=android)](https://jan.ai/) + +![image](https://cdn-uploads.huggingface.co/production/uploads/655e3b59d5c0d3db5359ca3c/A65FII_r3rAi9wZtK5P_v.png) + +## Overview + +**Jan-v3-4B-base-instruct** is a 4B-parameter model obtained via post-training distillation from a larger teacher, transferring capabilities while preserving general-purpose performance on standard benchmarks. The result is a compact, ownable base that is straightforward to fine-tune, broadly applicable and minimizing the usual capacity–capability trade-offs. + +Building on this base, **Jan-Code**, a code-tuned variant, **will be released soon.** + +## Model Overview + +This repo contains the BF16 version of **Jan-v3-4B-base-instruct**, which has the following features: +- Type: Causal Language Models +- Training Stage: Pretraining & Post-training +- Number of Parameters: 4B in total +- Number of Layers: 36 +- Number of Attention Heads (GQA): 32 for Q and 8 for KV +- Context Length: **262,144 natively**. + +**Intended Use** + +* A better small base for downstream work: improved instruction following out of the box, strong starting point for fine-tuning, and effective lightweight coding assistance. + +## Performance + +![image](https://cdn-uploads.huggingface.co/production/uploads/655e3b59d5c0d3db5359ca3c/IGuQdKZ0_IGIwL0Wkcasi.png) + +## Quick Start + +### Integration with Jan Apps + +Jan-v3 demo is hosted on **Jan Browser** at **[chat.jan.ai](https://chat.jan.ai/)**. It is also optimized for direct integration with [Jan Desktop](https://jan.ai/), select the model in the app to start using it. + + +### Local Deployment + +**Using vLLM:** +```bash +vllm serve janhq/Jan-v3-4B-base-instruct \ + --host 0.0.0.0 \ + --port 1234 \ + --enable-auto-tool-choice \ + --tool-call-parser hermes + +``` + +**Using llama.cpp:** +```bash +llama-server --model Jan-v3-4B-base-instruct-Q8_0.gguf \ + --host 0.0.0.0 \ + --port 1234 \ + --jinja \ + --no-context-shift +``` + +### Recommended Parameters +For optimal performance in agentic and general tasks, we recommend the following inference parameters: +```yaml +temperature: 0.7 +top_p: 0.8 +top_k: 20 +``` + +## 🤝 Community & Support + +- **Discussions**: [Hugging Face Community](https://huggingface.co/janhq/Jan-v2-VL-8B/discussions) +- **Jan App**: Learn more about the Jan App at [jan.ai](https://jan.ai/) + +## 📄 Citation +```bibtex +Updated Soon +```