From 395b77ec7592826bd66ea750ecb91e19afd639d4 Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Sat, 25 Jul 2026 16:58:06 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: Intel/llava-gemma-2b Source: Original Platform --- .gitattributes | 50 +++ LICENSE.md | 61 ++++ README.md | 206 +++++++++++ added_tokens.json | 3 + config.json | 47 +++ configuration.json | 1 + gaudi_config.json | 10 + generation_config.json | 7 + model-00001-of-00002.safetensors | 3 + model-00001-of-00003.safetensors | 3 + model-00002-of-00002.safetensors | 3 + model-00002-of-00003.safetensors | 3 + model-00003-of-00003.safetensors | 3 + model.safetensors.index.json | 566 +++++++++++++++++++++++++++++++ model_state_dict.bin | 3 + preprocessor_config.json | 45 +++ processing_llavagemma.py | 138 ++++++++ special_tokens_map.json | 34 ++ tokenizer.json | 3 + tokenizer.model | 3 + tokenizer_config.json | 79 +++++ trainer_state.json | 3 + training_args.bin | 3 + usage.py | 38 +++ 24 files changed, 1315 insertions(+) create mode 100644 .gitattributes create mode 100644 LICENSE.md create mode 100644 README.md create mode 100644 added_tokens.json create mode 100644 config.json create mode 100644 configuration.json create mode 100644 gaudi_config.json create mode 100644 generation_config.json create mode 100644 model-00001-of-00002.safetensors create mode 100644 model-00001-of-00003.safetensors create mode 100644 model-00002-of-00002.safetensors create mode 100644 model-00002-of-00003.safetensors create mode 100644 model-00003-of-00003.safetensors create mode 100644 model.safetensors.index.json create mode 100644 model_state_dict.bin create mode 100644 preprocessor_config.json create mode 100644 processing_llavagemma.py create mode 100644 special_tokens_map.json create mode 100644 tokenizer.json create mode 100644 tokenizer.model create mode 100644 tokenizer_config.json create mode 100644 trainer_state.json create mode 100644 training_args.bin create mode 100644 usage.py diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..6e0c316 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,50 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bin.* filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zstandard filter=lfs diff=lfs merge=lfs -text +*.tfevents* filter=lfs diff=lfs merge=lfs -text +*.db* filter=lfs diff=lfs merge=lfs -text +*.ark* filter=lfs diff=lfs merge=lfs -text +**/*ckpt*data* filter=lfs diff=lfs merge=lfs -text +**/*ckpt*.meta filter=lfs diff=lfs merge=lfs -text +**/*ckpt*.index filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.gguf* filter=lfs diff=lfs merge=lfs -text +*.ggml filter=lfs diff=lfs merge=lfs -text +*.llamafile* filter=lfs diff=lfs merge=lfs -text +*.pt2 filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text + +tokenizer.json filter=lfs diff=lfs merge=lfs -text +trainer_state.json filter=lfs diff=lfs merge=lfs -text \ No newline at end of file diff --git a/LICENSE.md b/LICENSE.md new file mode 100644 index 0000000..364ce31 --- /dev/null +++ b/LICENSE.md @@ -0,0 +1,61 @@ +Intel Research Use License Agreement + +By using or distributing any portion or element of the Material, You agree to be +bound by this Agreement. + +1. Definitions. +“Agreement” means the terms and conditions for use, reproduction and +distribution of the Material set forth herein. “Material” means model weights or +other software Intel makes available to you under this Agreement. “You” or +“Your” means you, or your employer or any other person or entity (if you are +entering into this Agreement on such person or entity’s behalf), of the age +required under applicable laws, rules or regulations to provide legal consent +and that has legal authority to bind your employer or such other person or +entity if you are entering in this Agreement on their behalf. + +2. License. +Intel grants to You, a limited, non-transferable, non-sublicensable, +non-exclusive, worldwide, royalty-free, license under Intel’s copyrights in the +Material, to use, reproduce, display, and distribute the Material solely for +research purposes. + + 2.1 Restrictions. Except as authorized above, You will not: (a) use Material in any other way; (b) reverse engineer, decompile, or disassemble the Material, or (c) use Material to violate or aid in the violation of any international human right. + + 2.2 No Implied License. Except for the express license in Section 2, Intel does not grant You any express or implied license under any legal theory. Any other licenses from Intel require additional consideration. Nothing in this Agreement requires Intel to grant any additional license. + +3. Third party programs. +Your use of certain third-party software with or within the Material is subject +to your compliance with licensing you obtain directly from that third-party. A +listing of any such third-party software may accompany the Material. + +4. No Warranty. +The Material is provided “as is,” without any express or implied warranty of any +kind including warranties of merchantability, non-infringement, title, or +fitness for a particular purpose. The Material may be pre-release and may not be +fully functional. Intel is not required to maintain, update, or support any +Material. You are solely responsible for determining the appropriateness of +using or distributing the Material and assume any risks associated with Your use +of the Material. + +5. Limitation on Liability. +IN NO EVENT WILL INTEL BE LIABLE TO YOU UNDER ANY THEORY OF LIABILITY FOR ANY +LOST PROFITS OR DAMAGES (INCLUDING BUT NOT LIMITED TO DIRECT, INDIRECT, +SPECULATIVE, SPECIAL OR CONSEQUENTIAL DAMAGES) WHATSOEVER ARISING OUT OF OR IN +CONNECTION WITH THIS AGREEMENT (EVEN IF INTEL HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH LOSSES OR DAMAGES). + +6. Term and Termination. +The term of this Agreement will commence upon Your acceptance of this Agreement +or access to the Material and will continue in full force and effect until +terminated in accordance with the terms and conditions herein. Intel may +terminate this Agreement if You are in breach of any term or condition of this +Agreement. Upon termination of this Agreement, You shall delete and cease use of +the Material. Sections 4, 5 and 7 will survive the termination of this +Agreement. + +7. Governing Law and Jurisdiction. +All disputes will be governed by the laws of the United States of America and +the State of Delaware without reference to conflict of law principles and +subject to the exclusive jurisdiction of the state or federal courts sitting in +the State of Delaware, and each party agrees that it submits to the personal +jurisdiction and venue of those courts and waives any objections. \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..a5c5c94 --- /dev/null +++ b/README.md @@ -0,0 +1,206 @@ +--- +language: +- en +license_name: intel-research-use-license +license_link: LICENSE.md +base_model: google/gemma-2b-it +tags: +- LLM +- Intel +model-index: +- name: llava-gemma-2b + results: + - task: + type: Large Language Model + name: Large Language Model + metrics: + - type: GQA + name: GQA + value: 0.531 + - type: MME Cog. + name: MME Cog. + value: 236 + - type: MME Per. + name: MME Per. + value: 1130 + - type: MM-Vet + name: MM-Vet + value: 17.7 + - type: POPE Acc. + name: POPE Acc. + value: 0.850 + - type: POPE F1 + name: POPE F1 + value: 0.839 + - type: VQAv2 + name: VQAv2 + value: 70.7 + - type: MMVP + name: MMVP + value: 0.287 + - type: ScienceQA Image + name: ScienceQA Image + value: 0.564 +library_name: transformers +pipeline_tag: image-text-to-text +--- + +## Model Details: LLaVA-Gemma-2b + +`llava-gemma-2b` is a large multimodal model (LMM) trained using the [LLaVA-v1.5 framework](https://arxiv.org/abs/2310.03744) with the 2-billion parameter [google/gemma-2b-it](https://huggingface.co/google/gemma-2b-it) model as language backbone and the CLIP-based vision encoder. + +| Model Details | Description | +| ----------- | ----------- | +| Authors | Intel: [Musashi Hinck*](https://huggingface.co/musashihinck), [Matthew L. Olson*](https://huggingface.co/matthewlyleolson), [David Cobbley](https://huggingface.co/djcobble), [Shao-Yen Tseng](https://huggingface.co/shaoyent), [Vasudev Lal](https://huggingface.co/vasudevlal) | +| Date | March 2024 | +| Version | 1 | +| Type | Large multimodal model (LMM) | +| Paper or Other Resources | [LLaVA-Gemma: Accelerating Multimodal Foundation Models with a Compact Language Model](https://arxiv.org/abs/2404.01331) | +| License | [Gemma](https://ai.google.dev/gemma/terms) | +| Questions or Comments | [Community Tab](https://huggingface.co/Intel/llava-gemma-2b/discussions) and [Intel DevHub Discord](https://discord.gg/rv2Gp55UJQ)| + +This model card was created by [Benjamin Consolvo](https://huggingface.co/bconsolvo) and the authors listed above. + +## Intended Use + +| Intended Use | Description | +| ----------- | ----------- | +| Primary intended uses | The model has been finetuned for multimodal benchmark evaluations, but can also be used as a multimodal chatbot. | +| Primary intended users | Anyone using or evaluating multimodal models. | +| Out-of-scope uses | This model is not intended for uses that require high levels of factuality, high stakes situations, mental health or medical applications, generating misinformation or disinformation, impersonating others, facilitating or inciting harassment or violence, any use that could lead to the violation of a human right under the UN Declaration of Human Rights. | + +### How to use + +Using `llava-gemma` requires a [modified preprocessor](./processing_llavagemma.py) if your transformers version is < 4.41.1 + +For current usage, see [`usage.py`](./usage.py) or the following code block: + +```python +import requests +from PIL import Image +from transformers import ( + LlavaForConditionalGeneration, + AutoTokenizer, + AutoProcessor, + CLIPImageProcessor +) +#In this repo, needed for version < 4.41.1 +#from processing_llavagemma import LlavaGemmaProcessor +#processor = LlavaGemmaProcessor( tokenizer=AutoTokenizer.from_pretrained(checkpoint), image_processor=CLIPImageProcessor.from_pretrained(checkpoint)) + +checkpoint = "Intel/llava-gemma-2b" + +# Load model +model = LlavaForConditionalGeneration.from_pretrained(checkpoint) +processor = AutoProcessor.from_pretrained(checkpoint) + +# Prepare inputs +# Use gemma chat template +prompt = processor.tokenizer.apply_chat_template( + [{'role': 'user', 'content': "\nWhat's the content of the image?"}], + tokenize=False, + add_generation_prompt=True +) +url = "https://www.ilankelman.org/stopsigns/australia.jpg" +image = Image.open(requests.get(url, stream=True).raw) +inputs = processor(text=prompt, images=image, return_tensors="pt") + +# Generate +generate_ids = model.generate(**inputs, max_length=30) +output = processor.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0] +print(output) +``` + +For straightforward use as a chatbot (without images), you can modify the last portion of code to the following: + +```python +# Prepare inputs +# Use gemma chat template +prompt = processor.tokenizer.apply_chat_template( + [{'role': 'user', 'content': "Summarize the following paragraph? In this paper, we introduced LLaVA-Gemma, a compact vision-language model leveraging the Gemma Large Language Model in two variants, Gemma-2B and Gemma-7B. Our work provides a unique opportunity for researchers to explore the trade-offs between computational efficiency and multimodal understanding in small-scale models. The availability of both variants allows for a comparative analysis that sheds light on how model size impacts performance in various tasks. Our evaluations demonstrate the versatility and effectiveness of LLaVA-Gemma across a range of datasets, highlighting its potential as a benchmark for future research in small-scale vision-language models. With these models, future practitioners can optimize the performance of small-scale multimodal models more directly."}], + tokenize=False, + add_generation_prompt=True +) +# url = "https://www.ilankelman.org/stopsigns/australia.jpg" +# image = Image.open(requests.get(url, stream=True).raw) +inputs = processor(text=prompt, images=None, return_tensors="pt") + +# Generate +generate_ids = model.generate(**inputs, max_length=300) +output = processor.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0] +print(output) +``` + +## Factors + +| Factors | Description | +| ----------- | ----------- | +| Groups | - | +| Instrumentation | - | +| Environment | Trained for 4 hours on 8 Intel Gaudi 2 AI accelerators. | +| Card Prompts | Model training and deployment on alternate hardware and software will change model performance | + +## Metrics + +| Metrics | Description | +| ----------- | ----------- | +| Model performance measures | We evaluate the LlaVA-Gemma models on a similar collection of benchmarks to other LMM works: GQA; MME; MM-Vet; POPE (accuracy and F1); VQAv2; MMVP; the image subset of ScienceQA. Our experiments provide insights into the efficacy of various design choices within the LLaVA framework. | +| Decision thresholds | - | +| Approaches to uncertainty and variability | - | + +## Training Data + +The model was trained using the LLaVA-v1.5 data mixture. This is listed as follows: + +- 558K filtered image-text pairs from LAION/CC/SBU, captioned by BLIP. +- 158K GPT-generated multimodal instruction-following data. +- 450K academic-task-oriented VQA data mixture. +- 40K ShareGPT data. + +## Quantitative Analyses + +Performance of LLaVA-Gemma models across seven benchmarks. Highlighted box indicates strongest performance amongst LLaVA-Gemma models. Bottom two rows show self-reported performance of Llava Phi-2 and LLaVA-v1.5 respectively. The bolded **gemma-2b-it** is the current model used here in this model card. + +| LM Backbone | Vision Model | Pretrained Connector | GQA | MME cognition | MME perception | MM-Vet | POPE accuracy | POPE F1 | VQAv2 | ScienceQA Image | MMVP | +| ----------- | ------------ | -------------------- | ----- | ------------- | -------------- | ------ | ------------- | ------- | ----- | --------------- | ----- | +| **gemma-2b-it** | CLIP | Yes | 0.531 | 236 | 1130 | 17.7 | 0.850 |0.839| 70.65 | 0.564 | 0.287 | +| gemma-2b-it | CLIP | No | 0.481 | 248 | 935 | 13.1 | 0.784 | 0.762 | 61.74 | 0.549 | 0.180 | +| gemma-2b-it | DinoV2 | Yes |0.587| 307| 1133 |19.1| 0.853 | 0.838 |71.37| 0.555 | 0.227 | +| gemma-2b-it | DinoV2 | No | 0.501 | 309| 959 | 14.5 | 0.793 | 0.772 | 61.65 | 0.568 | 0.180 | +| | | | | | | | | | | | | +| gemma-7b-it | CLIP | Yes | 0.472 | 253 | 895 | 18.2 | 0.848 | 0.829 | 68.7 | 0.625 | 0.327 | +| gemma-7b-it | CLIP | No | 0.472 | 278 | 857 | 19.1 | 0.782 | 0.734 | 65.1 | 0.636 | 0.240 | +| gemma-7b-it | DinoV2 | Yes | 0.519 | 257 | 1021 | 14.3 | 0.794 | 0.762 | 65.2 | 0.628 | 0.327 | +| gemma-7b-it | DinoV2 | No | 0.459 | 226 | 771 | 12.2 | 0.693 | 0.567 | 57.4 | 0.598 | 0.267 | +| | | | | | | | | | | | | +| Phi-2b | CLIP | Yes | - | - | 1335 | 28.9 | - | 0.850 | 71.4 | 0.684 | - | +| Llama-2-7b | CLIP | Yes | 0.620 | 348 | 1511 | 30.6 | 0.850 | 0.859 | 78.5 | 0.704 | 46.1 | + +## Ethical Considerations + +Intel is committed to respecting human rights and avoiding causing or contributing to adverse impacts on human rights. See [Intel’s Global Human Rights Principles](https://www.intel.com/content/dam/www/central-libraries/us/en/documents/policy-human-rights.pdf). Intel’s products and software are intended only to be used in applications that do not cause or contribute to adverse impacts on human rights. + +| Ethical Considerations | Description | +| ----------- | ----------- | +| Data | The model was trained using the LLaVA-v1.5 data mixture as described above. | +| Human life | The model is not intended to inform decisions central to human life or flourishing. | +| Mitigations | No additional risk mitigation strategies were considered during model development. | +| Risks and harms | This model has not been assessed for harm or biases, and should not be used for sensitive applications where it may cause harm. | +| Use cases | - | + +## Caveats and Recommendations + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. + +## Citation details +```bibtex +@misc{hinck2024llavagemma, + title={LLaVA-Gemma: Accelerating Multimodal Foundation Models with a Compact Language Model}, + author={Musashi Hinck and Matthew L. Olson and David Cobbley and Shao-Yen Tseng and Vasudev Lal}, + year={2024}, + eprint={2404.01331}, + url={https://arxiv.org/abs/2404.01331}, + archivePrefix={arXiv}, + primaryClass={cs.CL} +} +``` \ No newline at end of file diff --git a/added_tokens.json b/added_tokens.json new file mode 100644 index 0000000..039d1b5 --- /dev/null +++ b/added_tokens.json @@ -0,0 +1,3 @@ +{ + "": 256000 +} diff --git a/config.json b/config.json new file mode 100644 index 0000000..ddfbe34 --- /dev/null +++ b/config.json @@ -0,0 +1,47 @@ +{ + "_name_or_path": "./llava-gemma-2b-it", + "architectures": [ + "LlavaForConditionalGeneration" + ], + "ignore_index": -100, + "image_token_index": 256000, + "model_type": "llava", + "projector_hidden_act": "gelu", + "text_config": { + "_name_or_path": "google/gemma-2b-it", + "architectures": [ + "GemmaForCausalLM" + ], + "bos_token_id": 2, + "eos_token_id": 1, + "head_dim": 256, + "hidden_activation": "gelu", + "hidden_size": 2048, + "intermediate_size": 16384, + "max_position_embeddings": 8192, + "model_type": "gemma", + "num_attention_heads": 8, + "num_hidden_layers": 18, + "num_key_value_heads": 1, + "pad_token_id": 0, + "rope_scaling": null, + "tie_word_embeddings": true, + "torch_dtype": "bfloat16", + "vocab_size": 256064 + }, + "torch_dtype": "float32", + "transformers_version": "4.39.0.dev0", + "vision_config": { + "hidden_size": 1024, + "image_size": 336, + "intermediate_size": 4096, + "model_type": "clip_vision_model", + "num_attention_heads": 16, + "num_hidden_layers": 24, + "patch_size": 14, + "projection_dim": 768, + "vocab_size": 32000 + }, + "vision_feature_layer": -2, + "vision_feature_select_strategy": "default" +} \ No newline at end of file diff --git a/configuration.json b/configuration.json new file mode 100644 index 0000000..4aef15d --- /dev/null +++ b/configuration.json @@ -0,0 +1 @@ +{"framework": "pytorch", "task": "image-text-to-text", "allow_remote": true} \ No newline at end of file diff --git a/gaudi_config.json b/gaudi_config.json new file mode 100644 index 0000000..aa2b687 --- /dev/null +++ b/gaudi_config.json @@ -0,0 +1,10 @@ +{ + "autocast_bf16_ops": null, + "autocast_fp32_ops": null, + "optimum_version": "1.17.1", + "transformers_version": "4.39.0.dev0", + "use_dynamic_shapes": false, + "use_fused_adam": true, + "use_fused_clip_norm": true, + "use_torch_autocast": false +} diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..1f9785c --- /dev/null +++ b/generation_config.json @@ -0,0 +1,7 @@ +{ + "_from_model_config": true, + "bos_token_id": 2, + "eos_token_id": 1, + "pad_token_id": 0, + "transformers_version": "4.39.0.dev0" +} diff --git a/model-00001-of-00002.safetensors b/model-00001-of-00002.safetensors new file mode 100644 index 0000000..c5c3110 --- /dev/null +++ b/model-00001-of-00002.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3fb92b2c053bac391d8038134436d37d427c52ac7b8044c343163a68ce85fd91 +size 4945242264 diff --git a/model-00001-of-00003.safetensors b/model-00001-of-00003.safetensors new file mode 100644 index 0000000..f9111dd --- /dev/null +++ b/model-00001-of-00003.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a34f002270e12d50917ff5d307951ed107140530372c2d549aea081e327b4f6f +size 4964385864 diff --git a/model-00002-of-00002.safetensors b/model-00002-of-00002.safetensors new file mode 100644 index 0000000..36666b5 --- /dev/null +++ b/model-00002-of-00002.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b63c3d5c02d77f9f1fe0722c89230631a0451e86881c0267227767407a9117b4 +size 1735365528 diff --git a/model-00002-of-00003.safetensors b/model-00002-of-00003.safetensors new file mode 100644 index 0000000..f4c4b63 --- /dev/null +++ b/model-00002-of-00003.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:21d7fe224a3dced4a95cd2316b9d131a5c80705115849dd980eafbfe108d5e5c +size 4999820616 diff --git a/model-00003-of-00003.safetensors b/model-00003-of-00003.safetensors new file mode 100644 index 0000000..e10eda7 --- /dev/null +++ b/model-00003-of-00003.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:40c198b0d6b4170bd3d8eb708a66a672b76ae70eece7ab843b8ed3605d97c85c +size 1300294808 diff --git a/model.safetensors.index.json b/model.safetensors.index.json new file mode 100644 index 0000000..58eff6f --- /dev/null +++ b/model.safetensors.index.json @@ -0,0 +1,566 @@ +{ + "metadata": { + "total_size": 11264425984 + }, + "weight_map": { + "language_model.model.embed_tokens.weight": "model-00001-of-00003.safetensors", + "language_model.model.layers.0.input_layernorm.weight": "model-00001-of-00003.safetensors", + "language_model.model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "language_model.model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "language_model.model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "language_model.model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "language_model.model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "language_model.model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "language_model.model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "language_model.model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "language_model.model.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors", + "language_model.model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "language_model.model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "language_model.model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "language_model.model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "language_model.model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "language_model.model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "language_model.model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "language_model.model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "language_model.model.layers.10.input_layernorm.weight": "model-00002-of-00003.safetensors", + "language_model.model.layers.10.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "language_model.model.layers.10.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "language_model.model.layers.10.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "language_model.model.layers.10.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "language_model.model.layers.10.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "language_model.model.layers.10.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "language_model.model.layers.10.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "language_model.model.layers.10.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "language_model.model.layers.11.input_layernorm.weight": "model-00002-of-00003.safetensors", + "language_model.model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "language_model.model.layers.11.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "language_model.model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "language_model.model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "language_model.model.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "language_model.model.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "language_model.model.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "language_model.model.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "language_model.model.layers.12.input_layernorm.weight": "model-00002-of-00003.safetensors", + "language_model.model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "language_model.model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "language_model.model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "language_model.model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "language_model.model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "language_model.model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "language_model.model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "language_model.model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "language_model.model.layers.13.input_layernorm.weight": "model-00002-of-00003.safetensors", + "language_model.model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "language_model.model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "language_model.model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "language_model.model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "language_model.model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "language_model.model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "language_model.model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "language_model.model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "language_model.model.layers.14.input_layernorm.weight": "model-00002-of-00003.safetensors", + "language_model.model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "language_model.model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "language_model.model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "language_model.model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "language_model.model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "language_model.model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "language_model.model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "language_model.model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "language_model.model.layers.15.input_layernorm.weight": "model-00003-of-00003.safetensors", + "language_model.model.layers.15.mlp.down_proj.weight": "model-00003-of-00003.safetensors", + "language_model.model.layers.15.mlp.gate_proj.weight": "model-00003-of-00003.safetensors", + "language_model.model.layers.15.mlp.up_proj.weight": "model-00003-of-00003.safetensors", + "language_model.model.layers.15.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", + "language_model.model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "language_model.model.layers.15.self_attn.o_proj.weight": "model-00003-of-00003.safetensors", + "language_model.model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "language_model.model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "language_model.model.layers.16.input_layernorm.weight": "model-00003-of-00003.safetensors", + "language_model.model.layers.16.mlp.down_proj.weight": "model-00003-of-00003.safetensors", + "language_model.model.layers.16.mlp.gate_proj.weight": "model-00003-of-00003.safetensors", + "language_model.model.layers.16.mlp.up_proj.weight": "model-00003-of-00003.safetensors", + "language_model.model.layers.16.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", + "language_model.model.layers.16.self_attn.k_proj.weight": "model-00003-of-00003.safetensors", + "language_model.model.layers.16.self_attn.o_proj.weight": "model-00003-of-00003.safetensors", + "language_model.model.layers.16.self_attn.q_proj.weight": "model-00003-of-00003.safetensors", + "language_model.model.layers.16.self_attn.v_proj.weight": "model-00003-of-00003.safetensors", + "language_model.model.layers.17.input_layernorm.weight": "model-00003-of-00003.safetensors", + "language_model.model.layers.17.mlp.down_proj.weight": "model-00003-of-00003.safetensors", + "language_model.model.layers.17.mlp.gate_proj.weight": "model-00003-of-00003.safetensors", + "language_model.model.layers.17.mlp.up_proj.weight": "model-00003-of-00003.safetensors", + "language_model.model.layers.17.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", + "language_model.model.layers.17.self_attn.k_proj.weight": "model-00003-of-00003.safetensors", + "language_model.model.layers.17.self_attn.o_proj.weight": "model-00003-of-00003.safetensors", + "language_model.model.layers.17.self_attn.q_proj.weight": "model-00003-of-00003.safetensors", + "language_model.model.layers.17.self_attn.v_proj.weight": "model-00003-of-00003.safetensors", + "language_model.model.layers.2.input_layernorm.weight": "model-00001-of-00003.safetensors", + "language_model.model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "language_model.model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "language_model.model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "language_model.model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "language_model.model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "language_model.model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "language_model.model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "language_model.model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "language_model.model.layers.3.input_layernorm.weight": "model-00002-of-00003.safetensors", + "language_model.model.layers.3.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "language_model.model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "language_model.model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "language_model.model.layers.3.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "language_model.model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "language_model.model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "language_model.model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "language_model.model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "language_model.model.layers.4.input_layernorm.weight": "model-00002-of-00003.safetensors", + "language_model.model.layers.4.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "language_model.model.layers.4.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "language_model.model.layers.4.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "language_model.model.layers.4.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "language_model.model.layers.4.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "language_model.model.layers.4.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "language_model.model.layers.4.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "language_model.model.layers.4.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "language_model.model.layers.5.input_layernorm.weight": "model-00002-of-00003.safetensors", + "language_model.model.layers.5.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "language_model.model.layers.5.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "language_model.model.layers.5.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "language_model.model.layers.5.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "language_model.model.layers.5.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "language_model.model.layers.5.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "language_model.model.layers.5.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "language_model.model.layers.5.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "language_model.model.layers.6.input_layernorm.weight": "model-00002-of-00003.safetensors", + "language_model.model.layers.6.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "language_model.model.layers.6.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "language_model.model.layers.6.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "language_model.model.layers.6.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "language_model.model.layers.6.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "language_model.model.layers.6.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "language_model.model.layers.6.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "language_model.model.layers.6.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "language_model.model.layers.7.input_layernorm.weight": "model-00002-of-00003.safetensors", + "language_model.model.layers.7.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "language_model.model.layers.7.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "language_model.model.layers.7.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "language_model.model.layers.7.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "language_model.model.layers.7.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "language_model.model.layers.7.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "language_model.model.layers.7.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "language_model.model.layers.7.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "language_model.model.layers.8.input_layernorm.weight": "model-00002-of-00003.safetensors", + "language_model.model.layers.8.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "language_model.model.layers.8.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "language_model.model.layers.8.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "language_model.model.layers.8.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "language_model.model.layers.8.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "language_model.model.layers.8.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "language_model.model.layers.8.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "language_model.model.layers.8.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "language_model.model.layers.9.input_layernorm.weight": "model-00002-of-00003.safetensors", + "language_model.model.layers.9.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "language_model.model.layers.9.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "language_model.model.layers.9.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "language_model.model.layers.9.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "language_model.model.layers.9.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "language_model.model.layers.9.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "language_model.model.layers.9.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "language_model.model.layers.9.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "language_model.model.norm.weight": "model-00003-of-00003.safetensors", + "multi_modal_projector.linear_1.bias": "model-00001-of-00003.safetensors", + "multi_modal_projector.linear_1.weight": "model-00001-of-00003.safetensors", + "multi_modal_projector.linear_2.bias": "model-00001-of-00003.safetensors", + "multi_modal_projector.linear_2.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.embeddings.class_embedding": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.embeddings.patch_embedding.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.embeddings.position_embedding.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.0.layer_norm1.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.0.layer_norm1.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.0.layer_norm2.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.0.layer_norm2.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.0.mlp.fc1.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.0.mlp.fc1.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.0.mlp.fc2.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.0.mlp.fc2.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.0.self_attn.k_proj.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.0.self_attn.out_proj.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.0.self_attn.out_proj.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.0.self_attn.q_proj.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.0.self_attn.v_proj.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.1.layer_norm1.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.1.layer_norm1.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.1.layer_norm2.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.1.layer_norm2.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.1.mlp.fc1.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.1.mlp.fc1.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.1.mlp.fc2.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.1.mlp.fc2.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.1.self_attn.k_proj.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.1.self_attn.out_proj.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.1.self_attn.out_proj.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.1.self_attn.q_proj.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.1.self_attn.v_proj.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.10.layer_norm1.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.10.layer_norm1.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.10.layer_norm2.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.10.layer_norm2.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.10.mlp.fc1.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.10.mlp.fc1.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.10.mlp.fc2.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.10.mlp.fc2.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.10.self_attn.k_proj.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.10.self_attn.out_proj.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.10.self_attn.out_proj.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.10.self_attn.q_proj.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.10.self_attn.v_proj.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.11.layer_norm1.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.11.layer_norm1.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.11.layer_norm2.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.11.layer_norm2.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.11.mlp.fc1.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.11.mlp.fc1.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.11.mlp.fc2.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.11.mlp.fc2.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.11.self_attn.k_proj.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.11.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.11.self_attn.out_proj.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.11.self_attn.out_proj.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.11.self_attn.q_proj.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.11.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.11.self_attn.v_proj.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.11.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.12.layer_norm1.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.12.layer_norm1.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.12.layer_norm2.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.12.layer_norm2.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.12.mlp.fc1.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.12.mlp.fc1.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.12.mlp.fc2.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.12.mlp.fc2.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.12.self_attn.k_proj.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.12.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.12.self_attn.out_proj.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.12.self_attn.out_proj.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.12.self_attn.q_proj.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.12.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.12.self_attn.v_proj.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.12.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.13.layer_norm1.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.13.layer_norm1.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.13.layer_norm2.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.13.layer_norm2.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.13.mlp.fc1.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.13.mlp.fc1.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.13.mlp.fc2.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.13.mlp.fc2.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.13.self_attn.k_proj.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.13.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.13.self_attn.out_proj.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.13.self_attn.out_proj.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.13.self_attn.q_proj.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.13.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.13.self_attn.v_proj.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.13.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.14.layer_norm1.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.14.layer_norm1.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.14.layer_norm2.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.14.layer_norm2.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.14.mlp.fc1.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.14.mlp.fc1.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.14.mlp.fc2.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.14.mlp.fc2.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.14.self_attn.k_proj.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.14.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.14.self_attn.out_proj.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.14.self_attn.out_proj.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.14.self_attn.q_proj.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.14.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.14.self_attn.v_proj.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.14.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.15.layer_norm1.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.15.layer_norm1.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.15.layer_norm2.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.15.layer_norm2.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.15.mlp.fc1.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.15.mlp.fc1.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.15.mlp.fc2.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.15.mlp.fc2.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.15.self_attn.k_proj.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.15.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.15.self_attn.out_proj.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.15.self_attn.out_proj.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.15.self_attn.q_proj.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.15.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.15.self_attn.v_proj.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.15.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.16.layer_norm1.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.16.layer_norm1.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.16.layer_norm2.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.16.layer_norm2.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.16.mlp.fc1.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.16.mlp.fc1.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.16.mlp.fc2.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.16.mlp.fc2.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.16.self_attn.k_proj.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.16.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.16.self_attn.out_proj.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.16.self_attn.out_proj.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.16.self_attn.q_proj.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.16.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.16.self_attn.v_proj.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.16.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.17.layer_norm1.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.17.layer_norm1.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.17.layer_norm2.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.17.layer_norm2.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.17.mlp.fc1.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.17.mlp.fc1.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.17.mlp.fc2.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.17.mlp.fc2.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.17.self_attn.k_proj.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.17.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.17.self_attn.out_proj.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.17.self_attn.out_proj.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.17.self_attn.q_proj.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.17.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.17.self_attn.v_proj.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.17.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.18.layer_norm1.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.18.layer_norm1.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.18.layer_norm2.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.18.layer_norm2.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.18.mlp.fc1.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.18.mlp.fc1.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.18.mlp.fc2.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.18.mlp.fc2.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.18.self_attn.k_proj.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.18.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.18.self_attn.out_proj.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.18.self_attn.out_proj.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.18.self_attn.q_proj.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.18.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.18.self_attn.v_proj.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.18.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.19.layer_norm1.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.19.layer_norm1.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.19.layer_norm2.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.19.layer_norm2.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.19.mlp.fc1.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.19.mlp.fc1.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.19.mlp.fc2.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.19.mlp.fc2.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.19.self_attn.k_proj.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.19.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.19.self_attn.out_proj.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.19.self_attn.out_proj.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.19.self_attn.q_proj.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.19.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.19.self_attn.v_proj.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.19.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.2.layer_norm1.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.2.layer_norm1.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.2.layer_norm2.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.2.layer_norm2.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.2.mlp.fc1.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.2.mlp.fc1.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.2.mlp.fc2.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.2.mlp.fc2.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.2.self_attn.k_proj.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.2.self_attn.out_proj.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.2.self_attn.out_proj.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.2.self_attn.q_proj.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.2.self_attn.v_proj.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.20.layer_norm1.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.20.layer_norm1.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.20.layer_norm2.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.20.layer_norm2.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.20.mlp.fc1.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.20.mlp.fc1.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.20.mlp.fc2.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.20.mlp.fc2.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.20.self_attn.k_proj.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.20.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.20.self_attn.out_proj.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.20.self_attn.out_proj.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.20.self_attn.q_proj.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.20.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.20.self_attn.v_proj.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.20.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.21.layer_norm1.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.21.layer_norm1.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.21.layer_norm2.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.21.layer_norm2.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.21.mlp.fc1.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.21.mlp.fc1.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.21.mlp.fc2.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.21.mlp.fc2.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.21.self_attn.k_proj.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.21.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.21.self_attn.out_proj.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.21.self_attn.out_proj.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.21.self_attn.q_proj.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.21.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.21.self_attn.v_proj.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.21.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.22.layer_norm1.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.22.layer_norm1.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.22.layer_norm2.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.22.layer_norm2.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.22.mlp.fc1.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.22.mlp.fc1.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.22.mlp.fc2.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.22.mlp.fc2.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.22.self_attn.k_proj.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.22.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.22.self_attn.out_proj.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.22.self_attn.out_proj.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.22.self_attn.q_proj.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.22.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.22.self_attn.v_proj.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.22.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.23.layer_norm1.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.23.layer_norm1.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.23.layer_norm2.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.23.layer_norm2.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.23.mlp.fc1.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.23.mlp.fc1.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.23.mlp.fc2.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.23.mlp.fc2.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.23.self_attn.k_proj.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.23.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.23.self_attn.out_proj.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.23.self_attn.out_proj.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.23.self_attn.q_proj.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.23.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.23.self_attn.v_proj.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.23.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.3.layer_norm1.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.3.layer_norm1.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.3.layer_norm2.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.3.layer_norm2.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.3.mlp.fc1.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.3.mlp.fc1.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.3.mlp.fc2.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.3.mlp.fc2.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.3.self_attn.k_proj.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.3.self_attn.out_proj.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.3.self_attn.out_proj.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.3.self_attn.q_proj.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.3.self_attn.v_proj.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.4.layer_norm1.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.4.layer_norm1.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.4.layer_norm2.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.4.layer_norm2.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.4.mlp.fc1.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.4.mlp.fc1.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.4.mlp.fc2.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.4.mlp.fc2.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.4.self_attn.k_proj.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.4.self_attn.out_proj.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.4.self_attn.out_proj.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.4.self_attn.q_proj.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.4.self_attn.v_proj.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.5.layer_norm1.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.5.layer_norm1.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.5.layer_norm2.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.5.layer_norm2.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.5.mlp.fc1.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.5.mlp.fc1.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.5.mlp.fc2.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.5.mlp.fc2.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.5.self_attn.k_proj.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.5.self_attn.out_proj.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.5.self_attn.out_proj.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.5.self_attn.q_proj.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.5.self_attn.v_proj.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.6.layer_norm1.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.6.layer_norm1.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.6.layer_norm2.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.6.layer_norm2.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.6.mlp.fc1.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.6.mlp.fc1.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.6.mlp.fc2.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.6.mlp.fc2.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.6.self_attn.k_proj.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.6.self_attn.out_proj.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.6.self_attn.out_proj.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.6.self_attn.q_proj.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.6.self_attn.v_proj.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.7.layer_norm1.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.7.layer_norm1.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.7.layer_norm2.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.7.layer_norm2.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.7.mlp.fc1.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.7.mlp.fc1.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.7.mlp.fc2.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.7.mlp.fc2.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.7.self_attn.k_proj.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.7.self_attn.out_proj.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.7.self_attn.out_proj.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.7.self_attn.q_proj.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.7.self_attn.v_proj.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.8.layer_norm1.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.8.layer_norm1.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.8.layer_norm2.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.8.layer_norm2.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.8.mlp.fc1.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.8.mlp.fc1.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.8.mlp.fc2.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.8.mlp.fc2.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.8.self_attn.k_proj.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.8.self_attn.out_proj.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.8.self_attn.out_proj.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.8.self_attn.q_proj.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.8.self_attn.v_proj.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.9.layer_norm1.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.9.layer_norm1.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.9.layer_norm2.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.9.layer_norm2.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.9.mlp.fc1.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.9.mlp.fc1.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.9.mlp.fc2.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.9.mlp.fc2.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.9.self_attn.k_proj.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.9.self_attn.out_proj.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.9.self_attn.out_proj.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.9.self_attn.q_proj.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.9.self_attn.v_proj.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.encoder.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.post_layernorm.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.post_layernorm.weight": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.pre_layrnorm.bias": "model-00001-of-00003.safetensors", + "vision_tower.vision_model.pre_layrnorm.weight": "model-00001-of-00003.safetensors" + } +} diff --git a/model_state_dict.bin b/model_state_dict.bin new file mode 100644 index 0000000..45175ec --- /dev/null +++ b/model_state_dict.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4592258950a83b37959c2e0d068d2f64fc52811160d41e505b03b1a12fde5a57 +size 6239210215 diff --git a/preprocessor_config.json b/preprocessor_config.json new file mode 100644 index 0000000..cf5bb0c --- /dev/null +++ b/preprocessor_config.json @@ -0,0 +1,45 @@ +{ + "_valid_processor_keys": [ + "images", + "do_resize", + "size", + "resample", + "do_center_crop", + "crop_size", + "do_rescale", + "rescale_factor", + "do_normalize", + "image_mean", + "image_std", + "do_convert_rgb", + "return_tensors", + "data_format", + "input_data_format" + ], + "crop_size": { + "height": 336, + "width": 336 + }, + "do_center_crop": true, + "do_convert_rgb": true, + "do_normalize": true, + "do_rescale": true, + "do_resize": true, + "image_mean": [ + 0.48145466, + 0.4578275, + 0.40821073 + ], + "image_processor_type": "CLIPImageProcessor", + "image_std": [ + 0.26862954, + 0.26130258, + 0.27577711 + ], + "processor_class": "LlavaProcessor", + "resample": 3, + "rescale_factor": 0.00392156862745098, + "size": { + "shortest_edge": 336 + } +} diff --git a/processing_llavagemma.py b/processing_llavagemma.py new file mode 100644 index 0000000..983673b --- /dev/null +++ b/processing_llavagemma.py @@ -0,0 +1,138 @@ +# coding=utf-8 +# Copyright 2023 The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Processor class for Llava. +Modified to include support for Gemma tokenizer. +""" + + +from typing import List, Optional, Union + +from transformers.feature_extraction_utils import BatchFeature +from transformers.image_utils import ImageInput +from transformers.processing_utils import ProcessorMixin +from transformers.tokenization_utils_base import PaddingStrategy, PreTokenizedInput, TextInput, TruncationStrategy +from transformers.utils import TensorType + + +class LlavaGemmaProcessor(ProcessorMixin): + r""" + Constructs a Llava processor which wraps a Llava image processor and a Llava tokenizer into a single processor. + + [`LlavaProcessor`] offers all the functionalities of [`CLIPImageProcessor`] and [`LlamaTokenizerFast`]. See the + [`~LlavaProcessor.__call__`] and [`~LlavaProcessor.decode`] for more information. + + Args: + image_processor ([`CLIPImageProcessor`], *optional*): + The image processor is a required input. + tokenizer ([`LlamaTokenizerFast`], *optional*): + The tokenizer is a required input. + """ + + attributes = ["image_processor", "tokenizer"] + image_processor_class = "CLIPImageProcessor" + tokenizer_class = ("LlamaTokenizer", "LlamaTokenizerFast", + "GemmaTokenizer", "GemmaTokenizerFast") + + def __init__(self, image_processor=None, tokenizer=None): + super().__init__(image_processor, tokenizer) + + def __call__( + self, + text: Union[TextInput, PreTokenizedInput, List[TextInput], List[PreTokenizedInput]] = None, + images: ImageInput = None, + padding: Union[bool, str, PaddingStrategy] = False, + truncation: Union[bool, str, TruncationStrategy] = None, + max_length=None, + return_tensors: Optional[Union[str, TensorType]] = TensorType.PYTORCH, + ) -> BatchFeature: + """ + Main method to prepare for the model one or several sequences(s) and image(s). This method forwards the `text` + and `kwargs` arguments to LlamaTokenizerFast's [`~LlamaTokenizerFast.__call__`] if `text` is not `None` to encode + the text. To prepare the image(s), this method forwards the `images` and `kwrags` arguments to + CLIPImageProcessor's [`~CLIPImageProcessor.__call__`] if `images` is not `None`. Please refer to the doctsring + of the above two methods for more information. + + Args: + text (`str`, `List[str]`, `List[List[str]]`): + The sequence or batch of sequences to be encoded. Each sequence can be a string or a list of strings + (pretokenized string). If the sequences are provided as list of strings (pretokenized), you must set + `is_split_into_words=True` (to lift the ambiguity with a batch of sequences). + images (`PIL.Image.Image`, `np.ndarray`, `torch.Tensor`, `List[PIL.Image.Image]`, `List[np.ndarray]`, `List[torch.Tensor]`): + The image or batch of images to be prepared. Each image can be a PIL image, NumPy array or PyTorch + tensor. In case of a NumPy array/PyTorch tensor, each image should be of shape (C, H, W), where C is a + number of channels, H and W are image height and width. + padding (`bool`, `str` or [`~utils.PaddingStrategy`], *optional*, defaults to `False`): + Select a strategy to pad the returned sequences (according to the model's padding side and padding + index) among: + - `True` or `'longest'`: Pad to the longest sequence in the batch (or no padding if only a single + sequence if provided). + - `'max_length'`: Pad to a maximum length specified with the argument `max_length` or to the maximum + acceptable input length for the model if that argument is not provided. + - `False` or `'do_not_pad'` (default): No padding (i.e., can output a batch with sequences of different + lengths). + max_length (`int`, *optional*): + Maximum length of the returned list and optionally padding length (see above). + truncation (`bool`, *optional*): + Activates truncation to cut input sequences longer than `max_length` to `max_length`. + return_tensors (`str` or [`~utils.TensorType`], *optional*): + If set, will return tensors of a particular framework. Acceptable values are: + + - `'tf'`: Return TensorFlow `tf.constant` objects. + - `'pt'`: Return PyTorch `torch.Tensor` objects. + - `'np'`: Return NumPy `np.ndarray` objects. + - `'jax'`: Return JAX `jnp.ndarray` objects. + + Returns: + [`BatchFeature`]: A [`BatchFeature`] with the following fields: + + - **input_ids** -- List of token ids to be fed to a model. Returned when `text` is not `None`. + - **attention_mask** -- List of indices specifying which tokens should be attended to by the model (when + `return_attention_mask=True` or if *"attention_mask"* is in `self.model_input_names` and if `text` is not + `None`). + - **pixel_values** -- Pixel values to be fed to a model. Returned when `images` is not `None`. + """ + if images is not None: + pixel_values = self.image_processor(images, return_tensors=return_tensors)["pixel_values"] + else: + pixel_values = None + text_inputs = self.tokenizer( + text, return_tensors=return_tensors, padding=padding, truncation=truncation, max_length=max_length + ) + + return BatchFeature(data={**text_inputs, "pixel_values": pixel_values}) + + # Copied from transformers.models.clip.processing_clip.CLIPProcessor.batch_decode with CLIP->Llama + def batch_decode(self, *args, **kwargs): + """ + This method forwards all its arguments to LlamaTokenizerFast's [`~PreTrainedTokenizer.batch_decode`]. Please + refer to the docstring of this method for more information. + """ + return self.tokenizer.batch_decode(*args, **kwargs) + + # Copied from transformers.models.clip.processing_clip.CLIPProcessor.decode with CLIP->Llama + def decode(self, *args, **kwargs): + """ + This method forwards all its arguments to LlamaTokenizerFast's [`~PreTrainedTokenizer.decode`]. Please refer to + the docstring of this method for more information. + """ + return self.tokenizer.decode(*args, **kwargs) + + @property + # Copied from transformers.models.clip.processing_clip.CLIPProcessor.model_input_names + def model_input_names(self): + tokenizer_input_names = self.tokenizer.model_input_names + image_processor_input_names = self.image_processor.model_input_names + return list(dict.fromkeys(tokenizer_input_names + image_processor_input_names)) \ No newline at end of file diff --git a/special_tokens_map.json b/special_tokens_map.json new file mode 100644 index 0000000..8d6368f --- /dev/null +++ b/special_tokens_map.json @@ -0,0 +1,34 @@ +{ + "additional_special_tokens": [ + "", + "" + ], + "bos_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "unk_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } +} diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..73a394e --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ff3488032b3b615f80411104f84e694e60bc23ccb4b6011836173b43f828fed8 +size 17478113 diff --git a/tokenizer.model b/tokenizer.model new file mode 100644 index 0000000..796efe9 --- /dev/null +++ b/tokenizer.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:61a7b147390c64585d6c3543dd6fc636906c9af3865a5548f27f31aee1d4c8e2 +size 4241003 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..ec353f9 --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,79 @@ +{ + "add_bos_token": true, + "add_eos_token": false, + "added_tokens_decoder": { + "0": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "1": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "2": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "3": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "106": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "107": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "256000": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "additional_special_tokens": [ + "", + "" + ], + "bos_token": "", + "chat_template": "{{ bos_token }}{% if messages[0]['role'] == 'system' %}{{ raise_exception('System role not supported') }}{% endif %}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if (message['role'] == 'assistant') %}{% set role = 'model' %}{% else %}{% set role = message['role'] %}{% endif %}{{ '' + role + '\n' + message['content'] | trim + '\n' }}{% endfor %}{% if add_generation_prompt %}{{'model\n'}}{% endif %}", + "clean_up_tokenization_spaces": false, + "eos_token": "", + "legacy": null, + "model_max_length": 1000000000000000019884624838656, + "pad_token": "", + "processor_class": "LlavaGemmaProcessor", + "sp_model_kwargs": {}, + "spaces_between_special_tokens": false, + "tokenizer_class": "GemmaTokenizer", + "unk_token": "", + "use_default_system_prompt": false +} diff --git a/trainer_state.json b/trainer_state.json new file mode 100644 index 0000000..c77cb5a --- /dev/null +++ b/trainer_state.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4549962a519c313963156296814327cdf37bf8dbf902c0dcdce97808f202131f +size 1273682 diff --git a/training_args.bin b/training_args.bin new file mode 100644 index 0000000..25015bc --- /dev/null +++ b/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a8b841c47dbf60ce73eec7fd18249885409c079b26f693a5ac1690aac98c0970 +size 6840 diff --git a/usage.py b/usage.py new file mode 100644 index 0000000..fe5f4d4 --- /dev/null +++ b/usage.py @@ -0,0 +1,38 @@ +import transformers + +print(transformers.__version__) + +import requests +from PIL import Image +from transformers import ( + LlavaForConditionalGeneration, + AutoTokenizer, + CLIPImageProcessor +) +from processing_llavagemma import LlavaGemmaProcessor + +checkpoint = "Intel/llava-gemma-2b" + +model = LlavaForConditionalGeneration.from_pretrained(checkpoint) +processor = LlavaGemmaProcessor( + tokenizer=AutoTokenizer.from_pretrained(checkpoint), + image_processor=CLIPImageProcessor.from_pretrained(checkpoint) +) + +model.to('cuda') + + +prompt = processor.tokenizer.apply_chat_template( + [{'role': 'user', 'content': "What's the content of the image?"}], + tokenize=False, + add_generation_prompt=True +) +url = "https://www.ilankelman.org/stopsigns/australia.jpg" +image = Image.open(requests.get(url, stream=True).raw) +inputs = processor(text=prompt, images=image, return_tensors="pt") +inputs = {k: v.to('cuda') for k, v in inputs.items()} + +# Generate +generate_ids = model.generate(**inputs, max_length=30) +output = processor.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0] +print(output) \ No newline at end of file