From be50c7ad2cf8ef0dafdcd5d6bc7497bfae02718c Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Mon, 26 Oct 2026 14:12:16 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: xtuner/llava-phi-3-mini-gguf Source: Original Platform --- .gitattributes | 41 +++++++++++ OLLAMA_MODELFILE_F16 | 17 +++++ OLLAMA_MODELFILE_INT4 | 17 +++++ README.md | 116 +++++++++++++++++++++++++++++++ llava-phi-3-mini-f16.gguf | 3 + llava-phi-3-mini-int4.gguf | 3 + llava-phi-3-mini-mmproj-f16.gguf | 3 + 7 files changed, 200 insertions(+) create mode 100644 .gitattributes create mode 100644 OLLAMA_MODELFILE_F16 create mode 100644 OLLAMA_MODELFILE_INT4 create mode 100644 README.md create mode 100644 llava-phi-3-mini-f16.gguf create mode 100644 llava-phi-3-mini-int4.gguf create mode 100644 llava-phi-3-mini-mmproj-f16.gguf diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..84bd999 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,41 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +mmproj-model-f16.gguf filter=lfs diff=lfs merge=lfs -text +ggml-model-int4.gguf filter=lfs diff=lfs merge=lfs -text +ggml-model-f16.gguf filter=lfs diff=lfs merge=lfs -text +llava-phi-3-mini-f16.gguf filter=lfs diff=lfs merge=lfs -text +llava-phi-3-mini-int4.gguf filter=lfs diff=lfs merge=lfs -text +llava-phi-3-mini-mmproj-f16.gguf filter=lfs diff=lfs merge=lfs -text diff --git a/OLLAMA_MODELFILE_F16 b/OLLAMA_MODELFILE_F16 new file mode 100644 index 0000000..c502a94 --- /dev/null +++ b/OLLAMA_MODELFILE_F16 @@ -0,0 +1,17 @@ +FROM ./llava-phi-3-mini-f16.gguf +FROM ./llava-phi-3-mini-mmproj-f16.gguf +TEMPLATE """{{ if .System }}<|system|> +{{ .System }}<|end|> +{{ end }}{{ if .Prompt }}<|user|> +{{ .Prompt }}<|end|> +{{ end }}<|assistant|> +{{ .Response }}<|end|> +""" +PARAMETER stop "<|user|>" +PARAMETER stop "<|assistant|>" +PARAMETER stop "<|system|>" +PARAMETER stop "<|end|>" +PARAMETER stop "<|endoftext|>" + +PARAMETER num_keep 4 +PARAMETER num_ctx 4096 \ No newline at end of file diff --git a/OLLAMA_MODELFILE_INT4 b/OLLAMA_MODELFILE_INT4 new file mode 100644 index 0000000..8f07983 --- /dev/null +++ b/OLLAMA_MODELFILE_INT4 @@ -0,0 +1,17 @@ +FROM ./llava-phi-3-mini-int4.gguf +FROM ./llava-phi-3-mini-mmproj-f16.gguf +TEMPLATE """{{ if .System }}<|system|> +{{ .System }}<|end|> +{{ end }}{{ if .Prompt }}<|user|> +{{ .Prompt }}<|end|> +{{ end }}<|assistant|> +{{ .Response }}<|end|> +""" +PARAMETER stop "<|user|>" +PARAMETER stop "<|assistant|>" +PARAMETER stop "<|system|>" +PARAMETER stop "<|end|>" +PARAMETER stop "<|endoftext|>" + +PARAMETER num_keep 4 +PARAMETER num_ctx 4096 \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..72ee36b --- /dev/null +++ b/README.md @@ -0,0 +1,116 @@ +--- +datasets: +- Lin-Chen/ShareGPT4V +pipeline_tag: image-to-text +--- + +
+ + + +[![Generic badge](https://img.shields.io/badge/GitHub-%20XTuner-black.svg)](https://github.com/InternLM/xtuner) + + +
+ +## Model + +llava-phi-3-mini is a LLaVA model fine-tuned from [microsoft/Phi-3-mini-4k-instruct](https://huggingface.co/microsoft/Phi-3-mini-4k-instruct) and [CLIP-ViT-Large-patch14-336](https://huggingface.co/openai/clip-vit-large-patch14-336) with [ShareGPT4V-PT](https://huggingface.co/datasets/Lin-Chen/ShareGPT4V) and [InternVL-SFT](https://github.com/OpenGVLab/InternVL/tree/main/internvl_chat#prepare-training-datasets) by [XTuner](https://github.com/InternLM/xtuner). + +**Note: This model is in GGUF format.** + +Resources: + +- GitHub: [xtuner](https://github.com/InternLM/xtuner) +- Official LLaVA format model: [xtuner/llava-phi-3-mini](https://huggingface.co/xtuner/llava-phi-3-mini) +- HuggingFace LLaVA format model: [xtuner/llava-phi-3-mini-hf](https://huggingface.co/xtuner/llava-phi-3-mini-hf) +- XTuner LLaVA format model: [xtuner/llava-phi-3-mini-xtuner](https://huggingface.co/xtuner/llava-phi-3-mini-xtuner) + + +## Details + +| Model | Visual Encoder | Projector | Resolution | Pretraining Strategy | Fine-tuning Strategy | Pretrain Dataset | Fine-tune Dataset | Pretrain Epoch | Fine-tune Epoch | +| :-------------------- | ------------------: | --------: | ---------: | ---------------------: | ------------------------: | ------------------------: | -----------------------: | -------------- | --------------- | +| LLaVA-v1.5-7B | CLIP-L | MLP | 336 | Frozen LLM, Frozen ViT | Full LLM, Frozen ViT | LLaVA-PT (558K) | LLaVA-Mix (665K) | 1 | 1 | +| LLaVA-Llama-3-8B | CLIP-L | MLP | 336 | Frozen LLM, Frozen ViT | Full LLM, LoRA ViT | LLaVA-PT (558K) | LLaVA-Mix (665K) | 1 | 1 | +| LLaVA-Llama-3-8B-v1.1 | CLIP-L | MLP | 336 | Frozen LLM, Frozen ViT | Full LLM, LoRA ViT | ShareGPT4V-PT (1246K) | InternVL-SFT (1268K) | 1 | 1 | +| **LLaVA-Phi-3-mini** | CLIP-L | MLP | 336 | Frozen LLM, Frozen ViT | Full LLM, Full ViT | ShareGPT4V-PT (1246K) | InternVL-SFT (1268K) | 1 | 2 | + +## Results + +
+Image +
+ +| Model | MMBench Test (EN) | MMMU Val | SEED-IMG | AI2D Test | ScienceQA Test | HallusionBench aAcc | POPE | GQA | TextVQA | MME | MMStar | +| :-------------------- | :---------------: | :-------: | :------: | :-------: | :------------: | :-----------------: | :--: | :--: | :-----: | :------: | :----: | +| LLaVA-v1.5-7B | 66.5 | 35.3 | 60.5 | 54.8 | 70.4 | 44.9 | 85.9 | 62.0 | 58.2 | 1511/348 | 30.3 | +| LLaVA-Llama-3-8B | 68.9 | 36.8 | 69.8 | 60.9 | 73.3 | 47.3 | 87.2 | 63.5 | 58.0 | 1506/295 | 38.2 | +| LLaVA-Llama-3-8B-v1.1 | 72.3 | 37.1 | 70.1 | 70.0 | 72.9 | 47.7 | 86.4 | 62.6 | 59.0 | 1469/349 | 45.1 | +| **LLaVA-Phi-3-mini** | 69.2 | 41.4 | 70.0 | 69.3 | 73.7 | 49.8 | 87.3 | 61.5 | 57.8 | 1477/313 | 43.7 | + + +## Quickstart + +### Download models + +```bash +# mmproj +wget https://huggingface.co/xtuner/llava-phi-3-mini-gguf/resolve/main/llava-phi-3-mini-mmproj-f16.gguf + +# fp16 llm +wget https://huggingface.co/xtuner/llava-phi-3-mini-gguf/resolve/main/llava-phi-3-mini-f16.gguf + +# int4 llm +wget https://huggingface.co/xtuner/llava-phi-3-mini-gguf/resolve/main/llava-phi-3-mini-int4.gguf + +# (optional) ollama fp16 modelfile +wget https://huggingface.co/xtuner/llava-phi-3-mini-gguf/resolve/main/OLLAMA_MODELFILE_F16 + +# (optional) ollama int4 modelfile +wget https://huggingface.co/xtuner/llava-phi-3-mini-gguf/resolve/main/OLLAMA_MODELFILE_INT4 +``` + +### Chat by `ollama` + +Note: llava-phi-3-mini uses the `Phi-3-instruct` chat template. + +```bash +# fp16 +ollama create llava-phi3-f16 -f ./OLLAMA_MODELFILE_F16 +ollama run llava-phi3-f16 "xx.png Describe this image" + +# int4 +ollama create llava-phi3-int4 -f ./OLLAMA_MODELFILE_INT4 +ollama run llava-phi3-int4 "xx.png Describe this image" +``` + +### Chat by `./llava-cli` + +1. Build [llama.cpp](https://github.com/ggerganov/llama.cpp) ([docs](https://github.com/ggerganov/llama.cpp?tab=readme-ov-file#usage)) . +2. Build `./llava-cli` ([docs](https://github.com/ggerganov/llama.cpp/tree/master/examples/llava#usage)). + +Note: llava-phi-3-mini uses the `Phi-3-instruct` chat template. + +```bash +# fp16 +./llava-cli -m ./llava-phi-3-mini-f16.gguf --mmproj ./llava-phi-3-mini-mmproj-f16.gguf --image YOUR_IMAGE.jpg -c 4096 + +# int4 +./llava-cli -m ./llava-phi-3-mini-int4.gguf --mmproj ./llava-phi-3-mini-mmproj-f16.gguf --image YOUR_IMAGE.jpg -c 4096 +``` + +### Reproduce + +Please refer to [docs](https://github.com/InternLM/xtuner/tree/main/xtuner/configs/llava/phi3_mini_4k_instruct_clip_vit_large_p14_336#readme). + +## Citation + +```bibtex +@misc{2023xtuner, + title={XTuner: A Toolkit for Efficiently Fine-tuning LLM}, + author={XTuner Contributors}, + howpublished = {\url{https://github.com/InternLM/xtuner}}, + year={2023} +} +``` diff --git a/llava-phi-3-mini-f16.gguf b/llava-phi-3-mini-f16.gguf new file mode 100644 index 0000000..6be7e96 --- /dev/null +++ b/llava-phi-3-mini-f16.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ff9c8f58905d414ae8a375f88cdd8d25649a85e4e44952def4977c50d1daf289 +size 7643301856 diff --git a/llava-phi-3-mini-int4.gguf b/llava-phi-3-mini-int4.gguf new file mode 100644 index 0000000..9c85b69 --- /dev/null +++ b/llava-phi-3-mini-int4.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:377876be20bac24488716c04824ab3a6978900679b40013b0d2585004555e658 +size 2318919200 diff --git a/llava-phi-3-mini-mmproj-f16.gguf b/llava-phi-3-mini-mmproj-f16.gguf new file mode 100644 index 0000000..12f2ec1 --- /dev/null +++ b/llava-phi-3-mini-mmproj-f16.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:004fc09697203296f72321b296a8d48aade2d23e553cbfb1c1e6a0b5157a08d5 +size 607648960