初始化项目，由ModelHub XC社区提供模型
Model: Alibaba-DAMO-Academy/RynnBrain-4B Source: Original Platform
This commit is contained in:
ModelHub XC
36
.gitattributes
vendored
Normal file
36
.gitattributes
vendored
Normal file
@@ -0,0 +1,36 @@
|
||||
*.7z filter=lfs diff=lfs merge=lfs -text
|
||||
*.arrow filter=lfs diff=lfs merge=lfs -text
|
||||
*.bin filter=lfs diff=lfs merge=lfs -text
|
||||
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
||||
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
||||
*.ftz filter=lfs diff=lfs merge=lfs -text
|
||||
*.gz filter=lfs diff=lfs merge=lfs -text
|
||||
*.h5 filter=lfs diff=lfs merge=lfs -text
|
||||
*.joblib filter=lfs diff=lfs merge=lfs -text
|
||||
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
||||
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
||||
*.model filter=lfs diff=lfs merge=lfs -text
|
||||
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
||||
*.npy filter=lfs diff=lfs merge=lfs -text
|
||||
*.npz filter=lfs diff=lfs merge=lfs -text
|
||||
*.onnx filter=lfs diff=lfs merge=lfs -text
|
||||
*.ot filter=lfs diff=lfs merge=lfs -text
|
||||
*.parquet filter=lfs diff=lfs merge=lfs -text
|
||||
*.pb filter=lfs diff=lfs merge=lfs -text
|
||||
*.pickle filter=lfs diff=lfs merge=lfs -text
|
||||
*.pkl filter=lfs diff=lfs merge=lfs -text
|
||||
*.pt filter=lfs diff=lfs merge=lfs -text
|
||||
*.pth filter=lfs diff=lfs merge=lfs -text
|
||||
*.rar filter=lfs diff=lfs merge=lfs -text
|
||||
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
||||
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
||||
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
||||
*.tar filter=lfs diff=lfs merge=lfs -text
|
||||
*.tflite filter=lfs diff=lfs merge=lfs -text
|
||||
*.tgz filter=lfs diff=lfs merge=lfs -text
|
||||
*.wasm filter=lfs diff=lfs merge=lfs -text
|
||||
*.xz filter=lfs diff=lfs merge=lfs -text
|
||||
*.zip filter=lfs diff=lfs merge=lfs -text
|
||||
*.zst filter=lfs diff=lfs merge=lfs -text
|
||||
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
||||
tokenizer.json filter=lfs diff=lfs merge=lfs -text
|
||||
118
README.md
Normal file
118
README.md
Normal file
@@ -0,0 +1,118 @@
|
||||
---
|
||||
language:
|
||||
- en
|
||||
- zh
|
||||
library_name: transformers
|
||||
license: apache-2.0 # TODO: 如果不是 Apache-2.0 请改
|
||||
tags:
|
||||
- robotics
|
||||
- embodied-ai
|
||||
- egocentric
|
||||
- spatiotemporal
|
||||
- vision-language-model
|
||||
- video-understanding
|
||||
- grounding
|
||||
- planning
|
||||
- navigation
|
||||
- ocr
|
||||
- image-text-to-text
|
||||
- video-text-to-text
|
||||
- custom_code # 如果需要 trust_remote_code=True
|
||||
base_model: Qwen3-VL-8B-Instruct
|
||||
pipeline_tag: image-text-to-text
|
||||
---
|
||||
|
||||
<p align="center">
|
||||
<img src="https://cdn-uploads.huggingface.co/production/uploads/66974212a9e7257fc37798dc/8eIGdvaZEuZKTcsjIAXXK.png" width="350" style="margin-bottom: 0.2;"/>
|
||||
<p
|
||||
|
||||
<h2 align="center"><a href="https://github.com/alibaba-damo-academy/RynnBrain">RynnBrain: Open Embodied Foundation Models</a></h3>
|
||||
<h5 align="center"> If you like our project, please give us a star ⭐ on <a href="https://github.com/alibaba-damo-academy/RynnBrain">Github</a> for the latest update. </h5>
|
||||
|
||||
|
||||
---
|
||||
|
||||
## 📰 News
|
||||
- **[2026.02.02]** Release **RynnBrain family** weights and inference code.
|
||||
- **[2026.02.02]** Add cookbooks for cognition, localization, reasoning, and planning.
|
||||
|
||||
|
||||
## ✨ Introduction
|
||||
|
||||
RynnBrain aims to serve as a **physics-aware embodied brain**: it observes egocentric scenes, grounds language to physical space and time, and supports downstream robotic systems with reliable localization and planning outputs.
|
||||
|
||||
### Key Highlights
|
||||
- **Comprehensive egocentric understanding**
|
||||
Strong spatial comprehension and egocentric cognition across embodied QA, counting, OCR, and fine-grained video understanding.
|
||||
|
||||
- **Diverse spatiotemporal localization**
|
||||
Locates objects, target areas, and predicts trajectories across long episodic context, enabling global spatial awareness.
|
||||
|
||||
- **Physical-space grounded reasoning (RynnBrain family)**
|
||||
The broader RynnBrain family includes “Thinking” variants that interleave textual reasoning with spatial grounding to anchor reasoning in reality.
|
||||
|
||||
- **Physics-aware precise planning (RynnBrain family)**
|
||||
Integrates localized affordances/areas/objects into planning outputs to provide downstream VLA models with precise instructions.
|
||||
|
||||
## 🌎 Model Zoo
|
||||
|
||||
| Model | Base Model | Huggingface | Modelscope |
|
||||
| :--------------- | :------------------- | :---------: | :--------: |
|
||||
| RynnBrain-2B | Qwen3-VL-2B-Instruct | [Link](https://huggingface.co/Alibaba-DAMO-Academy/RynnBrain-2B) | [Link](https://www.modelscope.cn/models/DAMO_Academy/RynnBrain-2B) |
|
||||
| RynnBrain-4B (**This Checkpoint**) | Qwen3-VL-4B-Instruct | [Link](https://huggingface.co/Alibaba-DAMO-Academy/RynnBrain-4B) | [Link](https://www.modelscope.cn/models/DAMO_Academy/RynnBrain-4B) |
|
||||
| RynnBrain-8B | Qwen3-VL-8B-Instruct | [Link](https://huggingface.co/Alibaba-DAMO-Academy/RynnBrain-8B) | [Link](https://www.modelscope.cn/models/DAMO_Academy/RynnBrain-8B) |
|
||||
| RynnBrain-30B-A3B | Qwen3-VL-30B-A3B-Instruct | [Link](https://huggingface.co/Alibaba-DAMO-Academy/RynnBrain-30B-A3B) | [Link](https://www.modelscope.cn/models/DAMO_Academy/RynnBrain-30B-A3B) |
|
||||
| RynnBrain‑CoP-8B | RynnBrain-8B | [Link](https://huggingface.co/Alibaba-DAMO-Academy/RynnBrain-CoP-8B) | [Link](https://www.modelscope.cn/models/DAMO_Academy/RynnBrain-CoP-8B) |
|
||||
| RynnBrain‑Plan-8B | RynnBrain-8B | [Link](https://huggingface.co/Alibaba-DAMO-Academy/RynnBrain-Plan-8B) | [Link](https://www.modelscope.cn/models/DAMO_Academy/RynnBrain-Plan-8B) |
|
||||
| RynnBrain‑Plan-30B-A3B | RynnBrain-30B-A3B | [Link](https://huggingface.co/Alibaba-DAMO-Academy/RynnBrain-Plan-30B-A3B) | [Link](https://www.modelscope.cn/models/DAMO_Academy/RynnBrain-Plan-30B-A3B) |
|
||||
| RynnBrain‑Nav-8B | RynnBrain-8B | [Link](https://huggingface.co/Alibaba-DAMO-Academy/RynnBrain-Nav-8B) | [Link](https://www.modelscope.cn/models/DAMO_Academy/RynnBrain-Nav-8B) |
|
||||
|
||||
|
||||
|
||||
## 🚀 Main Results
|
||||
|
||||
<img width="500" alt="image" src="https://alibaba-damo-academy.github.io/RynnBrain.github.io/assets/result-8b.png">
|
||||
|
||||
## 🤖 Quick Start
|
||||
|
||||
Minimal dependencies:
|
||||
```shell
|
||||
pip install transformers==4.57.1
|
||||
```
|
||||
Run text generation:
|
||||
```python
|
||||
from transformers import AutoModelForImageTextToText
|
||||
|
||||
model = AutoModelForImageTextToText.from_pretrained("")
|
||||
...
|
||||
```
|
||||
## Cookbooks
|
||||
Checkout the [cookbooks](https://github.com/alibaba-damo-academy/RynnBrain/cookbooks) that showcase RynnBrain's capabilities in cognition, localization, reasoning, and planning.
|
||||
|
||||
|
||||
| Category | Cookbook name | Description |
|
||||
|----------------------|--------------------------------------------------------------------------------------------------|-------------|
|
||||
| Cognition | [1_spatial_understanding.ipynb](https://github.com/alibaba-damo-academy/RynnBrain/cookbooks/1_spatial_understanding.ipynb) | Shows the ability of model for spaital understanding in the video scene. |
|
||||
| Cognition | [2_object_understanding.ipynb](https://github.com/alibaba-damo-academy/RynnBrain/cookbooks/2_object_understanding.ipynb) | Shows how the model understands object categories, attributes, and relations and counting ability. |
|
||||
| Cognition | [3_ocr.ipynb](https://github.com/alibaba-damo-academy/RynnBrain/cookbooks/3_ocr.ipynb) | Examples of optical character recognition and text understanding in videos. |
|
||||
| Location | [4_object_location.ipynb](https://github.com/alibaba-damo-academy/RynnBrain/cookbooks/4_object_location.ipynb) | Locates specific objects with bounding boxes in an image or video based on instructions. |
|
||||
| Location | [5_area_location.ipynb](https://github.com/alibaba-damo-academy/RynnBrain/cookbooks/5_area_location.ipynb) | Identifies and marks specified regions by points in an image or video. |
|
||||
| Location | [6_affordance_location.ipynb](https://github.com/alibaba-damo-academy/RynnBrain/cookbooks/6_affordance_location.ipynb) | Finds areas or objects with specific affordances in an image of video. |
|
||||
| Location | [7_trajectory_location.ipynb](https://github.com/alibaba-damo-academy/RynnBrain/cookbooks/7_trajectory_location.ipynb) | Infers and annotates trajectories or motion paths in an image or video. |
|
||||
| Location | [8_grasp_pose.ipynb](https://github.com/alibaba-damo-academy/RynnBrain/cookbooks/8_grasp_pose.ipynb) | Present the model's abiltiy to predict robotic grasp poses from images. |
|
||||
|
||||
|
||||
## 📑 Citation
|
||||
|
||||
If you find RynnBrain useful for your research and applications, please cite using this BibTeX:
|
||||
|
||||
```bibtex
|
||||
@article{damo2026rynnbrain,
|
||||
title={RynnBrain: Open Embodied Foundation Models},
|
||||
author={Ronghao Dang, Jiayan Guo, Bohan Hou, Sicong Leng, Kehan Li, Xin Li, Jiangpin Liu, Yunxuan Mao, Zhikai Wang, Yuqian Yuan, Minghao Zhu, Xiao Lin, Yang Bai, Qian Jiang, Yaxi Zhao, Minghua Zeng, Junlong Gao, Yuming Jiang, Jun Cen, Siteng Huang, Liuyi Wang, Wenqiao Zhang, Chengju Liu, Jianfei Yang, Shijian Lu, Deli Zhao},
|
||||
journal={arXiv preprint arXiv:2602.14979v1},
|
||||
year={2026},
|
||||
url = {https://arxiv.org/abs/2602.14979v1}
|
||||
}
|
||||
|
||||
```
|
||||
28
added_tokens.json
Normal file
28
added_tokens.json
Normal file
@@ -0,0 +1,28 @@
|
||||
{
|
||||
"</think>": 151668,
|
||||
"</tool_call>": 151658,
|
||||
"</tool_response>": 151666,
|
||||
"<think>": 151667,
|
||||
"<tool_call>": 151657,
|
||||
"<tool_response>": 151665,
|
||||
"<|box_end|>": 151649,
|
||||
"<|box_start|>": 151648,
|
||||
"<|endoftext|>": 151643,
|
||||
"<|file_sep|>": 151664,
|
||||
"<|fim_middle|>": 151660,
|
||||
"<|fim_pad|>": 151662,
|
||||
"<|fim_prefix|>": 151659,
|
||||
"<|fim_suffix|>": 151661,
|
||||
"<|im_end|>": 151645,
|
||||
"<|im_start|>": 151644,
|
||||
"<|image_pad|>": 151655,
|
||||
"<|object_ref_end|>": 151647,
|
||||
"<|object_ref_start|>": 151646,
|
||||
"<|quad_end|>": 151651,
|
||||
"<|quad_start|>": 151650,
|
||||
"<|repo_name|>": 151663,
|
||||
"<|video_pad|>": 151656,
|
||||
"<|vision_end|>": 151653,
|
||||
"<|vision_pad|>": 151654,
|
||||
"<|vision_start|>": 151652
|
||||
}
|
||||
120
chat_template.jinja
Normal file
120
chat_template.jinja
Normal file
@@ -0,0 +1,120 @@
|
||||
{%- if tools %}
|
||||
{{- '<|im_start|>system\n' }}
|
||||
{%- if messages[0].role == 'system' %}
|
||||
{%- if messages[0].content is string %}
|
||||
{{- messages[0].content }}
|
||||
{%- else %}
|
||||
{%- for content in messages[0].content %}
|
||||
{%- if 'text' in content %}
|
||||
{{- content.text }}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{%- endif %}
|
||||
{{- '\n\n' }}
|
||||
{%- endif %}
|
||||
{{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
|
||||
{%- for tool in tools %}
|
||||
{{- "\n" }}
|
||||
{{- tool | tojson }}
|
||||
{%- endfor %}
|
||||
{{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
|
||||
{%- else %}
|
||||
{%- if messages[0].role == 'system' %}
|
||||
{{- '<|im_start|>system\n' }}
|
||||
{%- if messages[0].content is string %}
|
||||
{{- messages[0].content }}
|
||||
{%- else %}
|
||||
{%- for content in messages[0].content %}
|
||||
{%- if 'text' in content %}
|
||||
{{- content.text }}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{%- endif %}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- set image_count = namespace(value=0) %}
|
||||
{%- set video_count = namespace(value=0) %}
|
||||
{%- for message in messages %}
|
||||
{%- if message.role == "user" %}
|
||||
{{- '<|im_start|>' + message.role + '\n' }}
|
||||
{%- if message.content is string %}
|
||||
{{- message.content }}
|
||||
{%- else %}
|
||||
{%- for content in message.content %}
|
||||
{%- if content.type == 'image' or 'image' in content or 'image_url' in content %}
|
||||
{%- set image_count.value = image_count.value + 1 %}
|
||||
{%- if add_vision_id %}Picture {{ image_count.value }}: {% endif -%}
|
||||
<|vision_start|><|image_pad|><|vision_end|>
|
||||
{%- elif content.type == 'video' or 'video' in content %}
|
||||
{%- set video_count.value = video_count.value + 1 %}
|
||||
{%- if add_vision_id %}Video {{ video_count.value }}: {% endif -%}
|
||||
<|vision_start|><|video_pad|><|vision_end|>
|
||||
{%- elif 'text' in content %}
|
||||
{{- content.text }}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{%- endif %}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- elif message.role == "assistant" %}
|
||||
{{- '<|im_start|>' + message.role + '\n' }}
|
||||
{%- if message.content is string %}
|
||||
{{- message.content }}
|
||||
{%- else %}
|
||||
{%- for content_item in message.content %}
|
||||
{%- if 'text' in content_item %}
|
||||
{{- content_item.text }}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{%- endif %}
|
||||
{%- if message.tool_calls %}
|
||||
{%- for tool_call in message.tool_calls %}
|
||||
{%- if (loop.first and message.content) or (not loop.first) %}
|
||||
{{- '\n' }}
|
||||
{%- endif %}
|
||||
{%- if tool_call.function %}
|
||||
{%- set tool_call = tool_call.function %}
|
||||
{%- endif %}
|
||||
{{- '<tool_call>\n{"name": "' }}
|
||||
{{- tool_call.name }}
|
||||
{{- '", "arguments": ' }}
|
||||
{%- if tool_call.arguments is string %}
|
||||
{{- tool_call.arguments }}
|
||||
{%- else %}
|
||||
{{- tool_call.arguments | tojson }}
|
||||
{%- endif %}
|
||||
{{- '}\n</tool_call>' }}
|
||||
{%- endfor %}
|
||||
{%- endif %}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- elif message.role == "tool" %}
|
||||
{%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
|
||||
{{- '<|im_start|>user' }}
|
||||
{%- endif %}
|
||||
{{- '\n<tool_response>\n' }}
|
||||
{%- if message.content is string %}
|
||||
{{- message.content }}
|
||||
{%- else %}
|
||||
{%- for content in message.content %}
|
||||
{%- if content.type == 'image' or 'image' in content or 'image_url' in content %}
|
||||
{%- set image_count.value = image_count.value + 1 %}
|
||||
{%- if add_vision_id %}Picture {{ image_count.value }}: {% endif -%}
|
||||
<|vision_start|><|image_pad|><|vision_end|>
|
||||
{%- elif content.type == 'video' or 'video' in content %}
|
||||
{%- set video_count.value = video_count.value + 1 %}
|
||||
{%- if add_vision_id %}Video {{ video_count.value }}: {% endif -%}
|
||||
<|vision_start|><|video_pad|><|vision_end|>
|
||||
{%- elif 'text' in content %}
|
||||
{{- content.text }}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{%- endif %}
|
||||
{{- '\n</tool_response>' }}
|
||||
{%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{%- if add_generation_prompt %}
|
||||
{{- '<|im_start|>assistant\n' }}
|
||||
{%- endif %}
|
||||
63
config.json
Normal file
63
config.json
Normal file
@@ -0,0 +1,63 @@
|
||||
{
|
||||
"architectures": [
|
||||
"Qwen3VLForConditionalGeneration"
|
||||
],
|
||||
"image_token_id": 151655,
|
||||
"model_type": "qwen3_vl",
|
||||
"text_config": {
|
||||
"attention_bias": false,
|
||||
"attention_dropout": 0.0,
|
||||
"bos_token_id": 151643,
|
||||
"dtype": "bfloat16",
|
||||
"eos_token_id": 151645,
|
||||
"head_dim": 128,
|
||||
"hidden_act": "silu",
|
||||
"hidden_size": 2560,
|
||||
"initializer_range": 0.02,
|
||||
"intermediate_size": 9728,
|
||||
"max_position_embeddings": 262144,
|
||||
"model_type": "qwen3_vl_text",
|
||||
"num_attention_heads": 32,
|
||||
"num_hidden_layers": 36,
|
||||
"num_key_value_heads": 8,
|
||||
"rms_norm_eps": 1e-06,
|
||||
"rope_scaling": {
|
||||
"mrope_interleaved": true,
|
||||
"mrope_section": [
|
||||
24,
|
||||
20,
|
||||
20
|
||||
],
|
||||
"rope_type": "default"
|
||||
},
|
||||
"rope_theta": 5000000,
|
||||
"tie_word_embeddings": true,
|
||||
"use_cache": true,
|
||||
"vocab_size": 151936
|
||||
},
|
||||
"tie_word_embeddings": true,
|
||||
"transformers_version": "4.57.1",
|
||||
"video_token_id": 151656,
|
||||
"vision_config": {
|
||||
"deepstack_visual_indexes": [
|
||||
5,
|
||||
11,
|
||||
17
|
||||
],
|
||||
"depth": 24,
|
||||
"hidden_act": "gelu_pytorch_tanh",
|
||||
"hidden_size": 1024,
|
||||
"in_channels": 3,
|
||||
"initializer_range": 0.02,
|
||||
"intermediate_size": 4096,
|
||||
"model_type": "qwen3_vl",
|
||||
"num_heads": 16,
|
||||
"num_position_embeddings": 2304,
|
||||
"out_hidden_size": 2560,
|
||||
"patch_size": 16,
|
||||
"spatial_merge_size": 2,
|
||||
"temporal_patch_size": 2
|
||||
},
|
||||
"vision_end_token_id": 151653,
|
||||
"vision_start_token_id": 151652
|
||||
}
|
||||
14
generation_config.json
Normal file
14
generation_config.json
Normal file
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"do_sample": true,
|
||||
"eos_token_id": [
|
||||
151645,
|
||||
151645,
|
||||
151643
|
||||
],
|
||||
"pad_token_id": 151643,
|
||||
"repetition_penalty": 1.0,
|
||||
"temperature": 0.7,
|
||||
"top_k": 20,
|
||||
"top_p": 0.8,
|
||||
"transformers_version": "5.3.0"
|
||||
}
|
||||
151388
merges.txt
Normal file
151388
merges.txt
Normal file
File diff suppressed because it is too large
Load Diff
3
model.safetensors
Normal file
3
model.safetensors
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:1eebc842bcc0f28ab3571fcd326a82c15eeaf215300aa13217c04dbd11098eb7
|
||||
size 9653631848
|
||||
39
preprocessor_config.json
Normal file
39
preprocessor_config.json
Normal file
@@ -0,0 +1,39 @@
|
||||
{
|
||||
"crop_size": null,
|
||||
"data_format": "channels_first",
|
||||
"default_to_square": true,
|
||||
"device": null,
|
||||
"disable_grouping": null,
|
||||
"do_center_crop": null,
|
||||
"do_convert_rgb": true,
|
||||
"do_normalize": true,
|
||||
"do_pad": null,
|
||||
"do_rescale": true,
|
||||
"do_resize": true,
|
||||
"image_mean": [
|
||||
0.5,
|
||||
0.5,
|
||||
0.5
|
||||
],
|
||||
"image_processor_type": "Qwen2VLImageProcessorFast",
|
||||
"image_std": [
|
||||
0.5,
|
||||
0.5,
|
||||
0.5
|
||||
],
|
||||
"input_data_format": null,
|
||||
"max_pixels": null,
|
||||
"merge_size": 2,
|
||||
"min_pixels": null,
|
||||
"pad_size": null,
|
||||
"patch_size": 16,
|
||||
"processor_class": "Qwen3VLProcessor",
|
||||
"resample": 3,
|
||||
"rescale_factor": 0.00392156862745098,
|
||||
"return_tensors": null,
|
||||
"size": {
|
||||
"longest_edge": 16777216,
|
||||
"shortest_edge": 65536
|
||||
},
|
||||
"temporal_patch_size": 2
|
||||
}
|
||||
63
processor_config.json
Normal file
63
processor_config.json
Normal file
@@ -0,0 +1,63 @@
|
||||
{
|
||||
"image_processor": {
|
||||
"data_format": "channels_first",
|
||||
"do_convert_rgb": true,
|
||||
"do_normalize": true,
|
||||
"do_rescale": true,
|
||||
"do_resize": true,
|
||||
"image_mean": [
|
||||
0.5,
|
||||
0.5,
|
||||
0.5
|
||||
],
|
||||
"image_processor_type": "Qwen2VLImageProcessorFast",
|
||||
"image_std": [
|
||||
0.5,
|
||||
0.5,
|
||||
0.5
|
||||
],
|
||||
"merge_size": 2,
|
||||
"patch_size": 16,
|
||||
"resample": 3,
|
||||
"rescale_factor": 0.00392156862745098,
|
||||
"size": {
|
||||
"longest_edge": 16777216,
|
||||
"shortest_edge": 65536
|
||||
},
|
||||
"temporal_patch_size": 2
|
||||
},
|
||||
"processor_class": "Qwen3VLProcessor",
|
||||
"video_processor": {
|
||||
"data_format": "channels_first",
|
||||
"default_to_square": true,
|
||||
"do_convert_rgb": true,
|
||||
"do_normalize": true,
|
||||
"do_rescale": true,
|
||||
"do_resize": true,
|
||||
"do_sample_frames": true,
|
||||
"fps": 2,
|
||||
"image_mean": [
|
||||
0.5,
|
||||
0.5,
|
||||
0.5
|
||||
],
|
||||
"image_std": [
|
||||
0.5,
|
||||
0.5,
|
||||
0.5
|
||||
],
|
||||
"max_frames": 768,
|
||||
"merge_size": 2,
|
||||
"min_frames": 4,
|
||||
"patch_size": 16,
|
||||
"resample": 3,
|
||||
"rescale_factor": 0.00392156862745098,
|
||||
"return_metadata": false,
|
||||
"size": {
|
||||
"longest_edge": 25165824,
|
||||
"shortest_edge": 4096
|
||||
},
|
||||
"temporal_patch_size": 2,
|
||||
"video_processor_type": "Qwen3VLVideoProcessor"
|
||||
}
|
||||
}
|
||||
31
special_tokens_map.json
Normal file
31
special_tokens_map.json
Normal file
@@ -0,0 +1,31 @@
|
||||
{
|
||||
"additional_special_tokens": [
|
||||
"<|im_start|>",
|
||||
"<|im_end|>",
|
||||
"<|object_ref_start|>",
|
||||
"<|object_ref_end|>",
|
||||
"<|box_start|>",
|
||||
"<|box_end|>",
|
||||
"<|quad_start|>",
|
||||
"<|quad_end|>",
|
||||
"<|vision_start|>",
|
||||
"<|vision_end|>",
|
||||
"<|vision_pad|>",
|
||||
"<|image_pad|>",
|
||||
"<|video_pad|>"
|
||||
],
|
||||
"eos_token": {
|
||||
"content": "<|im_end|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
},
|
||||
"pad_token": {
|
||||
"content": "<|endoftext|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
}
|
||||
}
|
||||
BIN
tokenizer.json
(Stored with Git LFS)
Normal file
BIN
tokenizer.json
(Stored with Git LFS)
Normal file
Binary file not shown.
240
tokenizer_config.json
Normal file
240
tokenizer_config.json
Normal file
@@ -0,0 +1,240 @@
|
||||
{
|
||||
"add_bos_token": false,
|
||||
"add_prefix_space": false,
|
||||
"added_tokens_decoder": {
|
||||
"151643": {
|
||||
"content": "<|endoftext|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151644": {
|
||||
"content": "<|im_start|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151645": {
|
||||
"content": "<|im_end|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151646": {
|
||||
"content": "<|object_ref_start|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151647": {
|
||||
"content": "<|object_ref_end|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151648": {
|
||||
"content": "<|box_start|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151649": {
|
||||
"content": "<|box_end|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151650": {
|
||||
"content": "<|quad_start|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151651": {
|
||||
"content": "<|quad_end|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151652": {
|
||||
"content": "<|vision_start|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151653": {
|
||||
"content": "<|vision_end|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151654": {
|
||||
"content": "<|vision_pad|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151655": {
|
||||
"content": "<|image_pad|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151656": {
|
||||
"content": "<|video_pad|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151657": {
|
||||
"content": "<tool_call>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": false
|
||||
},
|
||||
"151658": {
|
||||
"content": "</tool_call>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": false
|
||||
},
|
||||
"151659": {
|
||||
"content": "<|fim_prefix|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": false
|
||||
},
|
||||
"151660": {
|
||||
"content": "<|fim_middle|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": false
|
||||
},
|
||||
"151661": {
|
||||
"content": "<|fim_suffix|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": false
|
||||
},
|
||||
"151662": {
|
||||
"content": "<|fim_pad|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": false
|
||||
},
|
||||
"151663": {
|
||||
"content": "<|repo_name|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": false
|
||||
},
|
||||
"151664": {
|
||||
"content": "<|file_sep|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": false
|
||||
},
|
||||
"151665": {
|
||||
"content": "<tool_response>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": false
|
||||
},
|
||||
"151666": {
|
||||
"content": "</tool_response>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": false
|
||||
},
|
||||
"151667": {
|
||||
"content": "<think>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": false
|
||||
},
|
||||
"151668": {
|
||||
"content": "</think>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": false
|
||||
}
|
||||
},
|
||||
"additional_special_tokens": [
|
||||
"<|im_start|>",
|
||||
"<|im_end|>",
|
||||
"<|object_ref_start|>",
|
||||
"<|object_ref_end|>",
|
||||
"<|box_start|>",
|
||||
"<|box_end|>",
|
||||
"<|quad_start|>",
|
||||
"<|quad_end|>",
|
||||
"<|vision_start|>",
|
||||
"<|vision_end|>",
|
||||
"<|vision_pad|>",
|
||||
"<|image_pad|>",
|
||||
"<|video_pad|>"
|
||||
],
|
||||
"bos_token": null,
|
||||
"clean_up_tokenization_spaces": false,
|
||||
"eos_token": "<|im_end|>",
|
||||
"errors": "replace",
|
||||
"extra_special_tokens": {},
|
||||
"model_max_length": 262144,
|
||||
"pad_token": "<|endoftext|>",
|
||||
"processor_class": "Qwen3VLProcessor",
|
||||
"split_special_tokens": false,
|
||||
"tokenizer_class": "Qwen2Tokenizer",
|
||||
"unk_token": null
|
||||
}
|
||||
41
video_preprocessor_config.json
Normal file
41
video_preprocessor_config.json
Normal file
@@ -0,0 +1,41 @@
|
||||
{
|
||||
"crop_size": null,
|
||||
"data_format": "channels_first",
|
||||
"default_to_square": true,
|
||||
"device": null,
|
||||
"do_center_crop": null,
|
||||
"do_convert_rgb": true,
|
||||
"do_normalize": true,
|
||||
"do_rescale": true,
|
||||
"do_resize": true,
|
||||
"do_sample_frames": true,
|
||||
"fps": 2,
|
||||
"image_mean": [
|
||||
0.5,
|
||||
0.5,
|
||||
0.5
|
||||
],
|
||||
"image_std": [
|
||||
0.5,
|
||||
0.5,
|
||||
0.5
|
||||
],
|
||||
"input_data_format": null,
|
||||
"max_frames": 768,
|
||||
"merge_size": 2,
|
||||
"min_frames": 4,
|
||||
"num_frames": null,
|
||||
"pad_size": null,
|
||||
"patch_size": 16,
|
||||
"processor_class": "Qwen3VLProcessor",
|
||||
"resample": 3,
|
||||
"rescale_factor": 0.00392156862745098,
|
||||
"return_metadata": false,
|
||||
"size": {
|
||||
"longest_edge": 25165824,
|
||||
"shortest_edge": 4096
|
||||
},
|
||||
"temporal_patch_size": 2,
|
||||
"video_metadata": null,
|
||||
"video_processor_type": "Qwen3VLVideoProcessor"
|
||||
}
|
||||
1
vocab.json
Normal file
1
vocab.json
Normal file
File diff suppressed because one or more lines are too long
Reference in New Issue
Block a user