491 lines
18 KiB
Markdown
491 lines
18 KiB
Markdown
---
|
||
base_model:
|
||
- Qwen/Qwen2.5-VL-3B-Instruct
|
||
datasets:
|
||
- remyxai/OpenSpaces
|
||
language:
|
||
- en
|
||
library_name: transformers
|
||
license_name: qwen-research
|
||
license_link: https://huggingface.co/Qwen/Qwen2.5-VL-3B-Instruct/blob/main/LICENSE
|
||
pipeline_tag: image-text-to-text
|
||
tags:
|
||
- remyx
|
||
- vqasynth
|
||
- spatial-reasoning
|
||
- multimodal
|
||
- vlm
|
||
- vision-language
|
||
- robotics
|
||
- distance-estimation
|
||
- embodied-ai
|
||
- quantitative-spatial-reasoning
|
||
new_version: remyxai/SpaceThinker-Qwen2.5VL-3B
|
||
model-index:
|
||
- name: SpaceQwen2.5-VL-3B-Instruct
|
||
results:
|
||
- task:
|
||
type: visual-question-answering
|
||
name: Spatial Reasoning
|
||
dataset:
|
||
name: 3DSRBench
|
||
type: benchmark
|
||
metrics:
|
||
- type: success_rate
|
||
name: Overall Success Rate
|
||
value: 0.515
|
||
results_by_subcategory:
|
||
- name: 3D Positional Relation / Orientation
|
||
success_rate: 0.4706
|
||
- name: Object Localization / 3D Localization
|
||
success_rate: 0.5629
|
||
- name: Object Properties / Size
|
||
success_rate: 0.5116
|
||
- task:
|
||
type: visual-question-answering
|
||
name: Spatial Reasoning
|
||
dataset:
|
||
name: BLINK
|
||
type: benchmark
|
||
metrics:
|
||
- type: success_rate
|
||
name: Overall Success Rate
|
||
value: 0.5
|
||
results_by_subcategory:
|
||
- name: 3D Positional Relation / Orientation
|
||
success_rate: 0.6503
|
||
- name: Counting / Object Counting
|
||
success_rate: 0.6083
|
||
- name: Depth and Distance / Relative
|
||
success_rate: 0.5161
|
||
- name: Object Localization / 2D Localization
|
||
success_rate: 0.4426
|
||
- name: Point and Object Tracking / Point Correspondence
|
||
success_rate: 0.2849
|
||
- task:
|
||
type: visual-question-answering
|
||
name: Spatial Reasoning
|
||
dataset:
|
||
name: MMIU
|
||
type: benchmark
|
||
metrics:
|
||
- type: success_rate
|
||
name: Overall Success Rate
|
||
value: 0.3045
|
||
results_by_subcategory:
|
||
- name: Camera and Image Transformation / 2D Transformation
|
||
success_rate: 0.245
|
||
- name: Camera and Image Transformation / 3D Camera Pose
|
||
success_rate: 0.215
|
||
- name: Camera and Image Transformation / Camera Motion
|
||
success_rate: 0.4436
|
||
- name: Depth and Distance / Absolute
|
||
success_rate: 0.265
|
||
- name: Object Localization / 3D Localization
|
||
success_rate: 0.48
|
||
- name: Point and Object Tracking / 3D Tracking
|
||
success_rate: 0.24
|
||
- name: Point and Object Tracking / Point Correspondence
|
||
success_rate: 0.28
|
||
- task:
|
||
type: visual-question-answering
|
||
name: Spatial Reasoning
|
||
dataset:
|
||
name: MMVP
|
||
type: benchmark
|
||
metrics:
|
||
- type: success_rate
|
||
name: Overall Success Rate
|
||
value: 0.5767
|
||
results_by_subcategory:
|
||
- name: Others / Miscellaneous
|
||
success_rate: 0.5767
|
||
- task:
|
||
type: visual-question-answering
|
||
name: Spatial Reasoning
|
||
dataset:
|
||
name: QSpatialBench-Plus
|
||
type: benchmark
|
||
metrics:
|
||
- type: success_rate
|
||
name: Overall Success Rate
|
||
value: 0.3663
|
||
results_by_subcategory:
|
||
- name: Depth and Distance / Absolute
|
||
success_rate: 0.3663
|
||
- task:
|
||
type: visual-question-answering
|
||
name: Spatial Reasoning
|
||
dataset:
|
||
name: QSpatialBench-ScanNet
|
||
type: benchmark
|
||
metrics:
|
||
- type: success_rate
|
||
name: Overall Success Rate
|
||
value: 0.33
|
||
results_by_subcategory:
|
||
- name: Depth and Distance / Absolute
|
||
success_rate: 0.216
|
||
- name: Object Properties / Size
|
||
success_rate: 0.4444
|
||
- task:
|
||
type: visual-question-answering
|
||
name: Spatial Reasoning
|
||
dataset:
|
||
name: RealWorldQA
|
||
type: benchmark
|
||
metrics:
|
||
- type: success_rate
|
||
name: Overall Success Rate
|
||
value: 0.4392
|
||
results_by_subcategory:
|
||
- name: Others / Miscellaneous
|
||
success_rate: 0.4392
|
||
- task:
|
||
type: visual-question-answering
|
||
name: Spatial Reasoning
|
||
dataset:
|
||
name: SpatialSense
|
||
type: benchmark
|
||
metrics:
|
||
- type: success_rate
|
||
name: Overall Success Rate
|
||
value: 0.6554
|
||
results_by_subcategory:
|
||
- name: 3D Positional Relation / Orientation
|
||
success_rate: 0.6554
|
||
- task:
|
||
type: visual-question-answering
|
||
name: Spatial Reasoning
|
||
dataset:
|
||
name: VGBench
|
||
type: benchmark
|
||
metrics:
|
||
- type: success_rate
|
||
name: Overall Success Rate
|
||
value: 0.2615
|
||
results_by_subcategory:
|
||
- name: Camera and Image Transformation / 2D Transformation
|
||
success_rate: 0.2277
|
||
- name: Camera and Image Transformation / 3D Camera Pose
|
||
success_rate: 0.2438
|
||
- name: Depth and Distance / Absolute
|
||
success_rate: 0.2696
|
||
- name: Depth and Distance / Relative
|
||
success_rate: 0.1945
|
||
- name: Object Localization / 3D Localization
|
||
success_rate: 0.3733
|
||
- name: Point and Object Tracking / 3D Tracking
|
||
success_rate: 0.2655
|
||
- task:
|
||
type: visual-question-answering
|
||
name: Spatial Reasoning
|
||
dataset:
|
||
name: VSI-Bench_8
|
||
type: benchmark
|
||
metrics:
|
||
- type: success_rate
|
||
name: Overall Success Rate
|
||
value: 0.2322
|
||
results_by_subcategory:
|
||
- name: 3D Positional Relation / Orientation
|
||
success_rate: 0.3843
|
||
- name: Counting / Object Counting
|
||
success_rate: 0.1715
|
||
- name: Depth and Distance / Absolute
|
||
success_rate: 0.0299
|
||
- name: Depth and Distance / Relative
|
||
success_rate: 0.3521
|
||
- name: Object Properties / Size
|
||
success_rate: 0.2323
|
||
- name: Others / Miscellaneous
|
||
success_rate: 0.2525
|
||
- task:
|
||
type: visual-question-answering
|
||
name: Spatial Reasoning
|
||
dataset:
|
||
name: VSR-ZeroShot
|
||
type: benchmark
|
||
metrics:
|
||
- type: success_rate
|
||
name: Overall Success Rate
|
||
value: 0.7373
|
||
results_by_subcategory:
|
||
- name: 3D Positional Relation / Orientation
|
||
success_rate: 0.7373
|
||
- task:
|
||
type: visual-question-answering
|
||
name: Spatial Reasoning
|
||
dataset:
|
||
name: cvbench
|
||
type: benchmark
|
||
metrics:
|
||
- type: success_rate
|
||
name: Overall Success Rate
|
||
value: 0.5179
|
||
results_by_subcategory:
|
||
- name: Counting / Object Counting
|
||
success_rate: 0.6168
|
||
- name: Depth and Distance / Relative
|
||
success_rate: 0.4925
|
||
- name: Object Localization / 3D Localization
|
||
success_rate: 0.4446
|
||
- task:
|
||
type: visual-question-answering
|
||
name: Spatial Reasoning
|
||
dataset:
|
||
name: spatialbench
|
||
type: benchmark
|
||
metrics:
|
||
- type: success_rate
|
||
name: Overall Success Rate
|
||
value: 0.4879
|
||
results_by_subcategory:
|
||
- name: 3D Positional Relation / Orientation
|
||
success_rate: 0.5294
|
||
- name: Counting / Object Counting
|
||
success_rate: 0.7
|
||
- name: Object Properties / Existence
|
||
success_rate: 0.45
|
||
- name: Object Properties / Reachability
|
||
success_rate: 0.5
|
||
- name: Object Properties / Size
|
||
success_rate: 0.25
|
||
---
|
||
|
||
<img src="https://cdn-uploads.huggingface.co/production/uploads/647777304ae93470ffc28913/v4edJliSy46xBA8g5ZXf8.png" width="500"/>
|
||
|
||
# SpaceQwen2.5-VL-3B-Instruct
|
||
|
||
The model was presented in the paper [OmniSpatial: Towards Comprehensive Spatial Reasoning Benchmark for Vision Language Models](https://huggingface.co/papers/2506.03135). More information can be found at the [project page](https://qizekun.github.io/omnispatial/).
|
||
|
||
|
||
- **Model Type:** Multimodal, Vision-Language Model
|
||
- **Architecture**: `Qwen2.5-VL-3B-Instruct`
|
||
- **Model Size:** 3.75B parameters (FP16)
|
||
- **Finetuned from:** Qwen/Qwen2.5-VL-3B-Instruct
|
||
- **Finetune Strategy:** LoRA (Low-Rank Adaptation)
|
||
- **License:** Apache-2.0
|
||
|
||
### Model Overview
|
||
|
||
This model uses data synthesis techniques and publicly available models to reproduce the work described in SpatialVLM to enhance the spatial reasoning of multimodal models.
|
||
With a pipeline of expert models, we can infer spatial relationships between objects in a scene to create a VQA dataset for spatial reasoning.
|
||
|
||
|
||
## Running SpaceQwen2.5-VL-3B-Instruct
|
||
|
||
### Ollama
|
||
To launch with ollama, run:
|
||
```bash
|
||
ollama run hf.co/remyxai/SpaceQwen2.5-VL-3B-Instruct:latest
|
||
```
|
||
|
||
### Transformers
|
||
|
||
Install qwen dependencies:
|
||
```
|
||
pip install qwen-vl-utils[decord]==0.0.8
|
||
```
|
||
|
||
To run inference on a sample image:
|
||
```python
|
||
from transformers import Qwen2_5_VLForConditionalGeneration, AutoTokenizer, AutoProcessor
|
||
from qwen_vl_utils import process_vision_info
|
||
|
||
model = Qwen2_5_VLForConditionalGeneration.from_pretrained(
|
||
"remyxai/SpaceQwen2.5-VL-3B-Instruct", torch_dtype="auto", device_map="auto"
|
||
)
|
||
processor = AutoProcessor.from_pretrained("remyxai/SpaceQwen2.5-VL-3B-Instruct")
|
||
|
||
messages = [
|
||
{
|
||
"role": "user",
|
||
"content": [
|
||
{
|
||
"type": "image",
|
||
"image": "https://raw.githubusercontent.com/remyxai/VQASynth/refs/heads/main/assets/warehouse_sample_2.jpeg",
|
||
},
|
||
{"type": "text", "text": "What is the height of the man in the red hat in feet?"},
|
||
],
|
||
}
|
||
]
|
||
|
||
# Preparation for inference
|
||
text = processor.apply_chat_template(
|
||
messages, tokenize=False, add_generation_prompt=True
|
||
)
|
||
image_inputs, video_inputs = process_vision_info(messages)
|
||
inputs = processor(
|
||
text=[text],
|
||
images=image_inputs,
|
||
videos=video_inputs,
|
||
padding=True,
|
||
return_tensors="pt",
|
||
)
|
||
inputs = inputs.to("cuda")
|
||
|
||
# Inference: Generation of the output
|
||
generated_ids = model.generate(**inputs, max_new_tokens=128)
|
||
generated_ids_trimmed = [
|
||
out_ids[len(in_ids) :] for in_ids, out_ids in zip(inputs.input_ids, generated_ids)
|
||
]
|
||
output_text = processor.batch_decode(
|
||
generated_ids_trimmed, skip_special_tokens=True, clean_up_tokenization_spaces=False
|
||
)
|
||
print(output_text)
|
||
```
|
||
|
||
### GGUF
|
||
|
||
Or run **SpaceQwen2.5-VL-3B-Instruct** using **llama.cpp**:
|
||
```bash
|
||
./llama-qwen2vl-cli -m /path/to/SpaceQwen2.5-VL-3B-Instruct/SpaceQwen2.5-VL-3B-Instruct-F16.gguf \
|
||
--mmproj /path/to/SpaceQwen2.5-VL-3B-Instruct/spaceqwen2.5-vl-3b-instruct-vision.gguf \
|
||
-p "What's the height of the man in the red hat?" \
|
||
--image /path/to/warehouse_sample_2.jpeg --threads 24 -ngl 99
|
||
```
|
||
|
||
|
||
## Dataset & Training
|
||
|
||
**SpaceQwen2.5-VL-3B-Instruct** uses LoRA to fine-tune [Qwen2.5-VL-3B-Instruct](https://huggingface.co/Qwen/Qwen2.5-VL-3B-Instruct) on the
|
||
[OpenSpaces](https://huggingface.co/datasets/salma-remyx/OpenSpaces) dataset.
|
||
|
||
**Dataset Summary**:
|
||
- ~10k synthetic spatial reasoning traces
|
||
- Question types: spatial relations (distances (units), above, left-of, contains, closest to)
|
||
- Format: image (RGB) + question + answer
|
||
|
||
- **Dataset:** [OpenSpaces](https://huggingface.co/datasets/remyxai/OpenSpaces)
|
||
- **Code:** [VQASynth](https://github.com/remyxai/VQASynth/tree/main)
|
||
- **Reference:** [SpatialVLM](https://spatial-vlm.github.io/)
|
||
|
||
Scripts for LoRA SFT available at [trl](https://github.com/huggingface/trl/blob/main/examples/scripts/sft_vlm.py)
|
||
|
||
|
||
## Model Evaluation
|
||
|
||
### SpatialScore
|
||
**SpaceQwen** shines in the 3D positional relations categories of the SpatialScore-Hard comparison featured in the table below:
|
||
|
||
|
||
|
||
Read more about the comprehensive spatial reasoning benchmark: [SpatialScore](https://haoningwu3639.github.io/SpatialScore/).
|
||
|
||
The following chart compares performance between **SpaceQwen** and **SpaceThinker** on the **SpatialScore** benchmarks sources.
|
||
|
||
<img src="https://cdn-uploads.huggingface.co/production/uploads/647777304ae93470ffc28913/M9DMOXkhef7-LBzGNxykm.png" alt="SpaceQwen_v_SpaceThinker" style="max-height: 250px;">
|
||
|
||
### OmniSpatial
|
||
|
||
**OmniSpatial** is another comprehensive spatial reasoning benchmark that assesses dynamic reasoning, complex spatial logic, spatial interaction, and perspective-taking capabilities.
|
||
|
||
|
||
Learn more about [OmniSpatial](https://qizekun.github.io/omnispatial/).
|
||
|
||
### SpaCE-10
|
||
|
||
|
||
|
||
| **Model** | **Overall** | **EQ** | **SQ** | **SA** | **OO** | **OS** | **EP** | **FR** | **SP** | **Source** |
|
||
|--------------------------|-------------|----------|----------|----------|----------|----------|----------|----------|----------|-------------|
|
||
| InternVL2.5-4B | **36.01** | **34.30**| 34.40 | 43.60 | 44.40 | 16.50 | **31.10**| **50.10**| **33.70**| Table |
|
||
| SpaceThinker | 32.72 | 32.73 | 24.81 | 47.26 | 50.33 | 33.63 | 9.25 | 37.54 | 26.25 | GPT Eval |
|
||
| SpaceOm | 32.32 | 32.47 | 24.81 | **47.63**| 50.00 | 32.52 | 9.12 | 37.04 | 25.00 | GPT Eval |
|
||
| **SpaceQwen** | 31.98 | 31.19 | 25.89 | 41.61 | **51.98**| **35.18**| 10.97 | 36.54 | 22.50 | GPT Eval |
|
||
| Qwen2.5-VL-3B-Instruct | 30.00 | 31.70 | **45.50**| 39.00 | 43.00 | 25.30 | 11.50 | 22.80 | 21.20 | Table |
|
||
|
||
|
||
**Legend:**
|
||
- EQ: Entity Quantification
|
||
- SQ: Scene Quantification
|
||
- SA: Size Assessment
|
||
- OO: Object-Object spatial relations
|
||
- OS: Object-Scene spatial relations
|
||
- EP: Entity Presence
|
||
- FR: Functional Reasoning
|
||
- SP: Spatial Planning
|
||
|
||
> ℹ️ Note: Scores for SpaceQwen, SpaceThinker, SpaceOm are generated via `gpt_eval_score` on single-choice (`*-single`) versions of the SpaCE-10 benchmark tasks. Other entries reflect leaderboard accuracy scores from the official SpaCE-10 evaluation table.
|
||
|
||
Read more about the [SpaCE-10 benchmark](https://arxiv.org/pdf/2506.07966v1) or see [results here](https://huggingface.co/datasets/salma-remyx/SpaceQwen_SpaCE-10_Results/blob/main/20250612_013312_results.json)
|
||
|
||
|
||
### SIRI-Bench
|
||
|
||
[SIRI-Bench](https://arxiv.org/pdf/2506.14512v1) is a video-based benchmark designed to evaluate complex spatial reasoning capabilities
|
||
|
||
|
||
|
||
|
||
### MindCube
|
||
|
||
[MindCube](https://huggingface.co/datasets/MLL-Lab/MindCube) is a benchmark for assessing [Spatial Mental Modeling from Limited Views](https://arxiv.org/pdf/2506.21458)
|
||
|
||
|
||
|
||
|
||
## ⚠️ Limitations & Ethical Considerations
|
||
|
||
- Performance may degrade in cluttered environments or camera perspective.
|
||
- This model was fine-tuned using synthetic reasoning over an internet image dataset.
|
||
- Multimodal biases inherent to the base model (Qwen2.5-VL) may persist.
|
||
- Not intended for use in safety-critical or legal decision-making.
|
||
|
||
> Users are encouraged to evaluate outputs critically and consider fine-tuning for domain-specific safety and performance.
|
||
|
||
## Citation
|
||
```
|
||
@article{chen2024spatialvlm,
|
||
title = {SpatialVLM: Endowing Vision-Language Models with Spatial Reasoning Capabilities},
|
||
author = {Chen, Boyuan and Xu, Zhuo and Kirmani, Sean and Ichter, Brian and Driess, Danny and Florence, Pete and Sadigh, Dorsa and Guibas, Leonidas and Xia, Fei},
|
||
journal = {arXiv preprint arXiv:2401.12168},
|
||
year = {2024},
|
||
url = {https://arxiv.org/abs/2401.12168},
|
||
}
|
||
|
||
@misc{qwen2.5-VL,
|
||
title = {Qwen2.5-VL},
|
||
url = {https://qwenlm.github.io/blog/qwen2.5-vl/},
|
||
author = {Qwen Team},
|
||
month = {January},
|
||
year = {2025}
|
||
}
|
||
|
||
@article{wu2025spatialscore,
|
||
author = {Wu, Haoning and Huang, Xiao and Chen, Yaohui and Zhang, Ya and Wang, Yanfeng and Xie, Weidi},
|
||
title = {SpatialScore: Towards Unified Evaluation for Multimodal Spatial Understanding},
|
||
journal = {arXiv preprint arXiv:2505.17012},
|
||
year = {2025},
|
||
}
|
||
|
||
@article{omnispatial25,
|
||
title = {OmniSpatial: Towards Comprehensive Spatial Reasoning Benchmark for Vision Language Models},
|
||
author = {Mengdi Jia and Zekun Qi and Shaochen Zhang and Wenyao Zhang and Xinqiang Yu and Jiawei He and He Wang and Li Yi},
|
||
journal = {arXiv preprint arXiv:2506.03135},
|
||
year = {2025}
|
||
}
|
||
|
||
@article{song2025siribench,
|
||
title = {{SIRI-Bench}: Challenging VLMs’ Spatial Intelligence through Complex Reasoning Tasks},
|
||
author = {Song, Zijian and Lin, Xiaoxin and Huang, Qiuming and Wang, Guangrun and Lin, Liang},
|
||
journal = {arXiv preprint arXiv:2506.14512},
|
||
year = {2025},
|
||
url = {https://arxiv.org/abs/2506.14512}
|
||
}
|
||
|
||
@misc{yin2025spatial,
|
||
title = {Spatial Mental Modeling from Limited Views},
|
||
author = {Baiqiao Yin and Qineng Wang and Pingyue Zhang and Jianshu Zhang
|
||
and Kangrui Wang and Zihan Wang and Jieyu Zhang
|
||
and Keshigeyan Chandrasegaran and Han Liu and Ranjay Krishna
|
||
and Saining Xie and Manling Li and Jiajun Wu and Li Fei-Fei},
|
||
year = {2025},
|
||
archivePrefix= {arXiv},
|
||
eprint = {2506.21458},
|
||
primaryClass = {cs.AI},
|
||
url = {https://arxiv.org/abs/2506.21458}
|
||
}
|
||
|
||
``` |