From 785cdad45467ec6862ec91adecdfc4977b5153eb Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Sat, 5 Sep 2026 17:50:18 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: llava-hf/LLaVA-NeXT-Video-7B-hf Source: Original Platform --- .gitattributes | 35 + README.md | 260 + added_tokens.json | 4 + chat_template.json | 3 + config.json | 69 + demo.png | Bin 0 -> 207895 bytes generation_config.json | 7 + model-00001-of-00003.safetensors | 3 + model-00002-of-00003.safetensors | 3 + model-00003-of-00003.safetensors | 3 + model.safetensors.index.json | 694 + preprocessor_config.json | 51 + processor_config.json | 8 + special_tokens_map.json | 30 + tokenizer.json | 93407 +++++++++++++++++++++++++++++ tokenizer.model | 3 + tokenizer_config.json | 66 + video_preprocessor_config.json | 63 + 18 files changed, 94709 insertions(+) create mode 100644 .gitattributes create mode 100644 README.md create mode 100644 added_tokens.json create mode 100644 chat_template.json create mode 100644 config.json create mode 100644 demo.png create mode 100644 generation_config.json create mode 100644 model-00001-of-00003.safetensors create mode 100644 model-00002-of-00003.safetensors create mode 100644 model-00003-of-00003.safetensors create mode 100644 model.safetensors.index.json create mode 100644 preprocessor_config.json create mode 100644 processor_config.json create mode 100644 special_tokens_map.json create mode 100644 tokenizer.json create mode 100644 tokenizer.model create mode 100644 tokenizer_config.json create mode 100644 video_preprocessor_config.json diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..a6344aa --- /dev/null +++ b/.gitattributes @@ -0,0 +1,35 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..3d4be69 --- /dev/null +++ b/README.md @@ -0,0 +1,260 @@ +--- +language: +- en +license: llama2 +pipeline_tag: video-text-to-text +datasets: +- lmms-lab/VideoChatGPT +--- + +# LLaVA-NeXT-Video Model Card + +Check out also the Google Colab demo to run Llava on a free-tier Google Colab instance: [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1CZggLHrjxMReG-FNOmqSOdi4z7NPq6SO?usp=sharing) + +Disclaimer: The team releasing LLaVa-NeXT-Video did not write a model card for this model so this model card has been written by the Hugging Face team. + +## 📄 Model details + +**Model type:** +LLaVA-Next-Video is an open-source chatbot trained by fine-tuning LLM on multimodal instruction-following data. The model is buit on top of LLaVa-NeXT by tuning on a mix of video and image data to achieve better video understanding capabilities. The videos were sampled uniformly to be 32 frames per clip. +The model is a current SOTA among open-source models on [VideoMME bench](https://arxiv.org/abs/2405.21075). +Base LLM: [lmsys/vicuna-7b-v1.5](https://huggingface.co/lmsys/vicuna-13b-v1.5) + +![llava_next_video_arch](demo.png) + + +**Model date:** +LLaVA-Next-Video-7B was trained in April 2024. + +**Paper or resources for more information:** https://github.com/LLaVA-VL/LLaVA-NeXT + + +## 📚 Training dataset + +### Image +- 558K filtered image-text pairs from LAION/CC/SBU, captioned by BLIP. +- 158K GPT-generated multimodal instruction-following data. +- 500K academic-task-oriented VQA data mixture. +- 50K GPT-4V data mixture. +- 40K ShareGPT data. + +### Video +- 100K VideoChatGPT-Instruct. + +## 📊 Evaluation dataset +A collection of 4 benchmarks, including 3 academic VQA benchmarks and 1 captioning benchmark. + + + +## 🚀 How to use the model + +First, make sure to have `transformers >= 4.42.0`. +The model supports multi-visual and multi-prompt generation. Meaning that you can pass multiple images/videos in your prompt. Make sure also to follow the correct prompt template (`USER: xxx\nASSISTANT:`) and add the token `` or `