1239 lines
50 KiB
Python
1239 lines
50 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||
|
||
# Adapted from
|
||
# https://github.com/huggingface/transformers/blob/main/src/transformers/models/qwen2_5_vl/modeling_qwen2_5_vl.py
|
||
# Copyright 2025 The vLLM team.
|
||
# Copyright 2025 The Qwen Team.
|
||
# Copyright 2025 The HuggingFace Inc. team.
|
||
# All rights reserved.
|
||
#
|
||
# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
|
||
# and OPT implementations in this library. It has been modified from its
|
||
# original forms to accommodate minor architectural differences compared
|
||
# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
|
||
#
|
||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||
# you may not use this file except in compliance with the License.
|
||
# You may obtain a copy of the License at
|
||
#
|
||
# http://www.apache.org/licenses/LICENSE-2.0
|
||
#
|
||
# Unless required by applicable law or agreed to in writing, software
|
||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||
# See the License for the specific language governing permissions and
|
||
# limitations under the License.
|
||
"""Inference-only Qwen2.5-VL model compatible with HuggingFace weights."""
|
||
from collections.abc import Iterable, Mapping
|
||
from functools import lru_cache, partial
|
||
from typing import Callable, Literal, Optional, TypedDict, Union
|
||
|
||
import torch
|
||
import torch.nn as nn
|
||
import torch.nn.functional as F
|
||
from einops import rearrange
|
||
from transformers import BatchFeature
|
||
from transformers.models.qwen2_5_vl import Qwen2_5_VLProcessor
|
||
from transformers.models.qwen2_5_vl.configuration_qwen2_5_vl import (
|
||
Qwen2_5_VLConfig, Qwen2_5_VLVisionConfig)
|
||
|
||
from vllm.config import VllmConfig
|
||
from vllm.distributed import parallel_state
|
||
from vllm.distributed import utils as dist_utils
|
||
from vllm.logger import init_logger
|
||
# from vllm.model_executor import SamplingMetadata
|
||
from vllm.model_executor.layers.activation import _ACTIVATION_REGISTRY
|
||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
|
||
QKVParallelLinear,
|
||
RowParallelLinear)
|
||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||
|
||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||
from vllm.model_executor.models.module_mapping import MultiModelKeys
|
||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||
from vllm.multimodal.inputs import MultiModalFieldConfig
|
||
from vllm.platforms import _Backend
|
||
from vllm.sequence import IntermediateTensors
|
||
from vllm.transformers_utils.config import uses_mrope
|
||
|
||
from vllm.model_executor.models.interfaces import (MultiModalEmbeddings, SupportsLoRA,
|
||
SupportsMultiModal, SupportsPP, SupportsQuant)
|
||
from .qwen2_vl import Qwen2VLDummyInputsBuilder as Qwen2_5_VLDummyInputsBuilder
|
||
from .qwen2_vl import (Qwen2VLMultiModalProcessor, Qwen2VLProcessingInfo,
|
||
apply_rotary_pos_emb_vision)
|
||
from vllm.model_executor.models.utils import (AutoWeightsLoader, WeightsMapper, cast_overflow_tensors,
|
||
init_vllm_registered_model, maybe_prefix,
|
||
merge_multimodal_embeddings)
|
||
from vllm.model_executor.models.vision import get_vit_attn_backend
|
||
|
||
logger = init_logger(__name__)
|
||
|
||
# === Vision Inputs === #
|
||
|
||
|
||
class Qwen2_5_VLImagePixelInputs(TypedDict):
|
||
type: Literal["pixel_values"]
|
||
pixel_values: torch.Tensor
|
||
"""Shape:
|
||
`(num_patches, num_channels * patch_size * patch_size)`
|
||
"""
|
||
|
||
image_grid_thw: torch.Tensor
|
||
"""Shape: `(num_images, 3)`
|
||
This should be in `(grid_t, grid_h, grid_w)` format.
|
||
"""
|
||
|
||
|
||
class Qwen2_5_VLImageEmbeddingInputs(TypedDict):
|
||
type: Literal["image_embeds"]
|
||
image_embeds: torch.Tensor
|
||
"""Supported types:
|
||
- list[`torch.Tensor`]: A list of tensors holding all images' features.
|
||
Each tensor holds an image's features.
|
||
- `torch.Tensor`: A tensor holding all images' features
|
||
(concatenation of all images' feature tensors).
|
||
|
||
Tensor shape: `(num_image_features, hidden_size)`
|
||
- `num_image_features` varies based on
|
||
the number and resolution of the images.
|
||
- `hidden_size` must match the hidden size of language model backbone.
|
||
"""
|
||
|
||
image_grid_thw: torch.Tensor
|
||
"""Shape: `(num_images, 3)`
|
||
This should be in `(grid_t, grid_h, grid_w)` format.
|
||
"""
|
||
|
||
|
||
Qwen2_5_VLImageInputs = Union[Qwen2_5_VLImagePixelInputs,
|
||
Qwen2_5_VLImageEmbeddingInputs]
|
||
|
||
|
||
class Qwen2_5_VLVideoPixelInputs(TypedDict):
|
||
type: Literal["pixel_values_videos"]
|
||
pixel_values_videos: torch.Tensor
|
||
"""Shape:
|
||
`(num_patches,
|
||
num_channels * temporal_patch_size * patch_size * patch_size)`
|
||
"""
|
||
|
||
video_grid_thw: torch.Tensor
|
||
"""Shape: `(num_videos, 3)`
|
||
|
||
This should be in `(grid_t, grid_h, grid_w)` format.
|
||
"""
|
||
|
||
second_per_grid_ts: torch.Tensor
|
||
"""
|
||
The video time interval (in seconds) for each grid along the temporal
|
||
dimension in the 3D position IDs. Returned when `videos` is not `None`.
|
||
"""
|
||
|
||
|
||
class Qwen2_5_VLVideoEmbeddingInputs(TypedDict):
|
||
type: Literal["video_embeds"]
|
||
video_embeds: torch.Tensor
|
||
"""Supported types:
|
||
- list[`torch.Tensor`]: A list of tensors holding all videos' features.
|
||
Each tensor holds an video's features.
|
||
- `torch.Tensor`: A tensor holding all videos' features
|
||
(concatenation of all videos' feature tensors).
|
||
|
||
Tensor shape: `(num_image_features, hidden_size)`
|
||
- `num_image_features` varies based on
|
||
the number and resolution of the videos.
|
||
- `hidden_size` must match the hidden size of language model backbone.
|
||
"""
|
||
|
||
video_grid_thw: torch.Tensor
|
||
"""Shape: `(num_videos, 3)`
|
||
This should be in `(grid_t, grid_h, grid_w)` format.
|
||
"""
|
||
|
||
|
||
Qwen2_5_VLVideoInputs = Union[Qwen2_5_VLVideoPixelInputs,
|
||
Qwen2_5_VLVideoEmbeddingInputs]
|
||
|
||
# === Vision Encoder === #
|
||
|
||
|
||
class Qwen2_5_VisionMLP(nn.Module):
|
||
|
||
def __init__(self,
|
||
in_features: int,
|
||
hidden_features: int,
|
||
bias: bool = False,
|
||
act_fn: Callable[[torch.Tensor], torch.Tensor] = F.silu,
|
||
quant_config: Optional[QuantizationConfig] = None,
|
||
prefix: str = ""):
|
||
super().__init__()
|
||
self.gate_proj = ColumnParallelLinear(in_features,
|
||
hidden_features,
|
||
bias=bias,
|
||
quant_config=quant_config,
|
||
prefix=f"{prefix}.gate_proj")
|
||
self.up_proj = ColumnParallelLinear(in_features,
|
||
hidden_features,
|
||
bias=bias,
|
||
quant_config=quant_config,
|
||
prefix=f"{prefix}.up_proj")
|
||
self.down_proj = RowParallelLinear(hidden_features,
|
||
in_features,
|
||
bias=bias,
|
||
quant_config=quant_config,
|
||
prefix=f"{prefix}.down_proj")
|
||
self.act_fn = act_fn
|
||
|
||
def forward(self, x: torch.Tensor):
|
||
x_gate, _ = self.gate_proj(x)
|
||
x_gate = self.act_fn(x_gate)
|
||
x_up, _ = self.up_proj(x)
|
||
x_down, _ = self.down_proj(x_gate * x_up)
|
||
return x_down
|
||
|
||
|
||
def all_gather_interleave(local_tensor, hidden_size: int, tp_size: int):
|
||
"""All-gather the input tensor interleavely across model parallel group."""
|
||
import torch.distributed as dist
|
||
gathered_tensors = [torch.zeros_like(local_tensor) for _ in range(tp_size)]
|
||
dist.all_gather(gathered_tensors,
|
||
local_tensor,
|
||
group=parallel_state.get_tp_group().device_group)
|
||
|
||
gathered_tensors_split = [
|
||
torch.split(tensor, hidden_size // tp_size, -1)
|
||
for tensor in gathered_tensors
|
||
]
|
||
ordered_tensors = [
|
||
tensor for pair in zip(*gathered_tensors_split) for tensor in pair
|
||
]
|
||
result_tensor = torch.cat(ordered_tensors, dim=-1)
|
||
return result_tensor
|
||
|
||
|
||
class Qwen2_5_VisionAttention(nn.Module):
|
||
"""
|
||
"""
|
||
def __init__(
|
||
self,
|
||
embed_dim: int,
|
||
num_heads: int,
|
||
projection_size: int,
|
||
quant_config: Optional[QuantizationConfig] = None,
|
||
prefix: str = "",
|
||
use_data_parallel: bool = False,
|
||
attn_backend: _Backend = _Backend.TORCH_SDPA,
|
||
use_upstream_fa: bool = False
|
||
) -> None:
|
||
"""
|
||
Initializes the Qwen2.5-VL module.
|
||
|
||
Args:
|
||
embed_dim (int): The embedding dimension for the input data.
|
||
num_heads (int): The number of attention heads.
|
||
projection_size (int): The size of the projection layer.
|
||
quant_config (Optional[QuantizationConfig], optional): The quantization configuration. Defaults to None.
|
||
prefix (str, optional): The prefix string for parameter names. Defaults to "".
|
||
|
||
Raises:
|
||
RuntimeError: If the attn backend is not supported.
|
||
"""
|
||
super().__init__()
|
||
# Per attention head and per partition values.
|
||
self.tp_size = parallel_state.get_tensor_model_parallel_world_size()
|
||
self.tp_rank = parallel_state.get_tensor_model_parallel_rank()
|
||
self.hidden_size_per_attention_head = dist_utils.divide(
|
||
projection_size, num_heads)
|
||
self.num_attention_heads_per_partition = dist_utils.divide(
|
||
num_heads, self.tp_size)
|
||
|
||
self.qkv = QKVParallelLinear(
|
||
hidden_size=embed_dim,
|
||
head_size=self.hidden_size_per_attention_head,
|
||
total_num_heads=num_heads,
|
||
total_num_kv_heads=num_heads,
|
||
bias=True,
|
||
quant_config=quant_config,
|
||
prefix=f"{prefix}.qkv")
|
||
self.proj = RowParallelLinear(input_size=projection_size,
|
||
output_size=embed_dim,
|
||
quant_config=quant_config,
|
||
prefix=f"{prefix}.proj")
|
||
|
||
# Detect attention implementation.
|
||
self.attn_backend = _Backend.FLASH_ATTN
|
||
if self.attn_backend not in {
|
||
_Backend.FLASH_ATTN, _Backend.TORCH_SDPA, _Backend.XFORMERS
|
||
}:
|
||
raise RuntimeError(
|
||
f"Qwen2.5-VL does not support {self.attn_backend} backend now."
|
||
)
|
||
|
||
def split_qkv(self, qkv: torch.Tensor) -> tuple[torch.Tensor, ...]:
|
||
# [s, b, 3 * head * head_dim]
|
||
seq_len, bs, _ = qkv.shape
|
||
if self.tp_size > 1:
|
||
qkv = all_gather_interleave(qkv, self.qkv.hidden_size,
|
||
self.tp_size)
|
||
|
||
# [s, b, 3 * head * head_dim] -> 3 * [s, b, head * head_dim]
|
||
q, k, v = qkv.chunk(3, dim=2)
|
||
|
||
# 3 * [s, b, head * head_dim]
|
||
if self.tp_size > 1:
|
||
splitter = partial(dist_utils.split_tensor_along_last_dim,
|
||
num_partitions=self.tp_size)
|
||
q = splitter(q)[self.tp_rank]
|
||
k = splitter(k)[self.tp_rank]
|
||
v = splitter(v)[self.tp_rank]
|
||
|
||
# 3 * [s, b, head * head_dim] -> 3 * [s, b, head, head_dim]
|
||
new_shape = (seq_len, bs, self.num_attention_heads_per_partition,
|
||
self.hidden_size_per_attention_head)
|
||
q, k, v = (x.view(*new_shape) for x in (q, k, v))
|
||
return q, k, v
|
||
|
||
def forward(
|
||
self,
|
||
x: torch.Tensor,
|
||
cu_seqlens: torch.Tensor,
|
||
rotary_pos_emb: torch.Tensor,
|
||
max_seqlen: Optional[int] = None, # Only used for Flash Attention
|
||
seqlens: Optional[list[int]] = None, # Only used for xFormers
|
||
) -> torch.Tensor:
|
||
# [s, b, c] --> [s, b, head * 3 * head_dim]
|
||
x, _ = self.qkv(x)
|
||
|
||
# [s, b, 3 * head * head_dim] -> 3 * [s, b, head, head_dim]
|
||
q, k, v = self.split_qkv(x)
|
||
batch_size = q.shape[1]
|
||
|
||
q, k, v = (rearrange(x, "s b ... -> b s ...").contiguous()
|
||
for x in (q, k, v))
|
||
if rotary_pos_emb is not None:
|
||
q = apply_rotary_pos_emb_vision(q, rotary_pos_emb)
|
||
k = apply_rotary_pos_emb_vision(k, rotary_pos_emb)
|
||
|
||
if self.attn_backend == _Backend.FLASH_ATTN:
|
||
# from vllm_flash_attn.flash_attn_interface import (
|
||
# flash_attn_varlen_func)
|
||
from flash_attn import flash_attn_varlen_func
|
||
|
||
q, k, v = (rearrange(x, "b s ... -> (b s) ...") for x in [q, k, v])
|
||
|
||
output = flash_attn_varlen_func(q,
|
||
k,
|
||
v,
|
||
cu_seqlens_q=cu_seqlens,
|
||
cu_seqlens_k=cu_seqlens,
|
||
max_seqlen_q=max_seqlen,
|
||
max_seqlen_k=max_seqlen,
|
||
dropout_p=0,
|
||
causal=False)
|
||
|
||
context_layer = rearrange(output,
|
||
"(b s) ... -> b s ...",
|
||
b=batch_size)
|
||
elif self.attn_backend == _Backend.TORCH_SDPA:
|
||
# Execute attention entry by entry for speed & less VRAM.
|
||
outputs = []
|
||
for i in range(1, len(cu_seqlens)):
|
||
start_idx = cu_seqlens[i - 1]
|
||
end_idx = cu_seqlens[i]
|
||
q_i = q[:, start_idx:end_idx]
|
||
k_i = k[:, start_idx:end_idx]
|
||
v_i = v[:, start_idx:end_idx]
|
||
q_i, k_i, v_i = (rearrange(x, "b s h d -> b h s d")
|
||
for x in [q_i, k_i, v_i])
|
||
output_i = F.scaled_dot_product_attention(q_i,
|
||
k_i,
|
||
v_i,
|
||
dropout_p=0.0)
|
||
output_i = rearrange(output_i, "b h s d -> b s h d ")
|
||
outputs.append(output_i)
|
||
context_layer = torch.cat(outputs, dim=1)
|
||
elif self.attn_backend == _Backend.XFORMERS:
|
||
from xformers import ops as xops
|
||
from xformers.ops.fmha.attn_bias import BlockDiagonalMask
|
||
|
||
attn_bias = BlockDiagonalMask.from_seqlens(q_seqlen=seqlens,
|
||
kv_seqlen=None,
|
||
device=q.device)
|
||
|
||
context_layer = xops.memory_efficient_attention_forward(
|
||
q, k, v, attn_bias=attn_bias, p=0, scale=None)
|
||
context_layer = rearrange(context_layer,
|
||
"b s h d -> s b (h d)").contiguous()
|
||
|
||
output, _ = self.proj(context_layer)
|
||
return output
|
||
|
||
|
||
class Qwen2_5_VisionBlock(nn.Module):
|
||
|
||
def __init__(
|
||
self,
|
||
dim: int,
|
||
num_heads: int,
|
||
mlp_hidden_dim: int,
|
||
act_fn: Callable[[torch.Tensor], torch.Tensor] = F.silu,
|
||
norm_layer: Optional[Callable[[int], nn.Module]] = None,
|
||
quant_config: Optional[QuantizationConfig] = None,
|
||
prefix: str = "",
|
||
) -> None:
|
||
super().__init__()
|
||
if norm_layer is None:
|
||
norm_layer = partial(nn.LayerNorm, eps=1e-6)
|
||
self.norm1 = norm_layer(dim)
|
||
self.norm2 = norm_layer(dim)
|
||
self.attn = Qwen2_5_VisionAttention(embed_dim=dim,
|
||
num_heads=num_heads,
|
||
projection_size=dim,
|
||
quant_config=quant_config,
|
||
prefix=f"{prefix}.attn")
|
||
self.mlp = Qwen2_5_VisionMLP(dim,
|
||
mlp_hidden_dim,
|
||
act_fn=act_fn,
|
||
bias=True,
|
||
quant_config=quant_config,
|
||
prefix=f"{prefix}.mlp")
|
||
|
||
def forward(
|
||
self,
|
||
x: torch.Tensor,
|
||
cu_seqlens: torch.Tensor,
|
||
rotary_pos_emb: torch.Tensor,
|
||
max_seqlen: Optional[int] = None, # Only used for Flash Attention
|
||
seqlens: Optional[list[int]] = None, # Only used for xFormers
|
||
) -> torch.Tensor:
|
||
x = x + self.attn(self.norm1(x),
|
||
cu_seqlens=cu_seqlens,
|
||
rotary_pos_emb=rotary_pos_emb,
|
||
max_seqlen=max_seqlen,
|
||
seqlens=seqlens)
|
||
|
||
x = x + self.mlp(self.norm2(x))
|
||
return x
|
||
|
||
|
||
class Qwen2_5_VisionPatchEmbed(nn.Module):
|
||
|
||
def __init__(
|
||
self,
|
||
patch_size: int = 14,
|
||
temporal_patch_size: int = 2,
|
||
in_channels: int = 3,
|
||
hidden_size: int = 1152,
|
||
) -> None:
|
||
super().__init__()
|
||
self.patch_size = patch_size
|
||
self.temporal_patch_size = temporal_patch_size
|
||
self.hidden_size = hidden_size
|
||
|
||
kernel_size = (temporal_patch_size, patch_size, patch_size)
|
||
self.proj = nn.Conv3d(in_channels,
|
||
hidden_size,
|
||
kernel_size=kernel_size,
|
||
stride=kernel_size,
|
||
bias=False)
|
||
|
||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||
L, C = x.shape
|
||
x = x.view(L, -1, self.temporal_patch_size, self.patch_size,
|
||
self.patch_size)
|
||
x = self.proj(x).view(L, self.hidden_size)
|
||
return x
|
||
|
||
|
||
class Qwen2_5_VisionPatchMerger(nn.Module):
|
||
|
||
def __init__(
|
||
self,
|
||
d_model: int,
|
||
context_dim: int,
|
||
norm_layer: Optional[Callable[[int], nn.Module]] = None,
|
||
spatial_merge_size: int = 2,
|
||
quant_config: Optional[QuantizationConfig] = None,
|
||
prefix: str = "",
|
||
) -> None:
|
||
super().__init__()
|
||
self.hidden_size = context_dim * (spatial_merge_size**2)
|
||
if norm_layer is None:
|
||
norm_layer = partial(nn.LayerNorm, eps=1e-6)
|
||
self.ln_q = norm_layer(context_dim)
|
||
self.mlp = nn.ModuleList([
|
||
ColumnParallelLinear(self.hidden_size,
|
||
self.hidden_size,
|
||
bias=True,
|
||
quant_config=quant_config,
|
||
prefix=f"{prefix}.mlp.0"),
|
||
nn.GELU(),
|
||
RowParallelLinear(self.hidden_size,
|
||
d_model,
|
||
bias=True,
|
||
quant_config=quant_config,
|
||
prefix=f"{prefix}.mlp.2"),
|
||
])
|
||
|
||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||
x = self.ln_q(x)
|
||
x = x.view(-1, self.hidden_size)
|
||
|
||
mlp_fc1, mlp_act, mlp_fc2 = self.mlp
|
||
x_parallel, _ = mlp_fc1(x)
|
||
x_parallel = mlp_act(x_parallel)
|
||
out, _ = mlp_fc2(x_parallel)
|
||
return out
|
||
|
||
|
||
class Qwen2_5_VisionRotaryEmbedding(nn.Module):
|
||
|
||
def __init__(self, dim: int, theta: float = 10000.0) -> None:
|
||
"""
|
||
初始化函数，用于创建一个实例。
|
||
|
||
Args:
|
||
dim (int): 维度大小，表示输入的特征向量长度。
|
||
theta (float, optional, default=10000.0): 参数，控制频谱分布的平滑程度，默认为10000.0。
|
||
|
||
Returns:
|
||
None: 不返回任何值，直接创建一个实例。
|
||
"""
|
||
super().__init__()
|
||
self.dim = dim
|
||
self.theta = theta
|
||
inv_freq = 1.0 / (theta**(
|
||
torch.arange(0, dim, 2, dtype=torch.float) / dim))
|
||
self.register_buffer("inv_freq", inv_freq, persistent=False)
|
||
self._seq_len_cached = 0
|
||
self._freqs_cached = None
|
||
|
||
def update_freqs_cache(self, seqlen: int) -> None:
|
||
if seqlen > self._seq_len_cached:
|
||
seqlen *= 2
|
||
self._seq_len_cached = seqlen
|
||
self.inv_freq = 1.0 / (self.theta**(torch.arange(
|
||
0, self.dim, 2, dtype=torch.float, device=self.inv_freq.device)
|
||
/ self.dim))
|
||
seq = torch.arange(seqlen,
|
||
device=self.inv_freq.device,
|
||
dtype=self.inv_freq.dtype)
|
||
freqs = torch.outer(seq, self.inv_freq)
|
||
self._freqs_cached = freqs
|
||
|
||
def forward(self, seqlen: int) -> torch.Tensor:
|
||
self.update_freqs_cache(seqlen)
|
||
return self._freqs_cached[:seqlen]
|
||
|
||
|
||
class Qwen2_5_VisionTransformer(nn.Module):
|
||
|
||
def __init__(
|
||
self,
|
||
vision_config: Qwen2_5_VLVisionConfig,
|
||
norm_eps: float = 1e-6,
|
||
quant_config: Optional[QuantizationConfig] = None,
|
||
prefix: str = "",
|
||
) -> None:
|
||
"""
|
||
Args:
|
||
vision_config (Qwen2_5_VLVisionConfig): config of the Vision Transformer model.
|
||
norm_eps (float, optional, default=1e-6): Epsilon added to the norm computation.
|
||
Defaults to 1e-6.
|
||
quant_config (Optional[QuantizationConfig], optional): Config for post-training quantization.
|
||
Defaults to None.
|
||
prefix (str, optional): Prefix string for module names.
|
||
Defaults to "".
|
||
"""
|
||
super().__init__()
|
||
|
||
patch_size = vision_config.patch_size
|
||
temporal_patch_size = vision_config.temporal_patch_size
|
||
in_channels = vision_config.in_channels
|
||
depth = vision_config.depth
|
||
self.hidden_size = vision_config.hidden_size
|
||
self.num_heads = vision_config.num_heads
|
||
|
||
# args for get_window_index_thw
|
||
self.window_size = vision_config.window_size
|
||
self.patch_size = vision_config.patch_size
|
||
self.spatial_merge_size = vision_config.spatial_merge_size
|
||
self.fullatt_block_indexes = vision_config.fullatt_block_indexes
|
||
self.spatial_merge_unit = self.spatial_merge_size**2
|
||
|
||
self.patch_embed = Qwen2_5_VisionPatchEmbed(
|
||
patch_size=patch_size,
|
||
temporal_patch_size=temporal_patch_size,
|
||
in_channels=in_channels,
|
||
hidden_size=self.hidden_size,
|
||
)
|
||
|
||
norm_layer = partial(RMSNorm, eps=norm_eps)
|
||
head_dim = self.hidden_size // self.num_heads
|
||
self.rotary_pos_emb = Qwen2_5_VisionRotaryEmbedding(head_dim // 2)
|
||
|
||
self.blocks = nn.ModuleList([
|
||
Qwen2_5_VisionBlock(
|
||
dim=self.hidden_size,
|
||
num_heads=self.num_heads,
|
||
mlp_hidden_dim=vision_config.intermediate_size,
|
||
act_fn=_ACTIVATION_REGISTRY[vision_config.hidden_act],
|
||
norm_layer=norm_layer,
|
||
quant_config=quant_config,
|
||
prefix=f"{prefix}.blocks.{layer_idx}")
|
||
for layer_idx in range(depth)
|
||
])
|
||
self.merger = Qwen2_5_VisionPatchMerger(
|
||
d_model=vision_config.out_hidden_size,
|
||
context_dim=self.hidden_size,
|
||
norm_layer=norm_layer,
|
||
spatial_merge_size=self.spatial_merge_size,
|
||
quant_config=quant_config,
|
||
prefix=f"{prefix}.merger",
|
||
)
|
||
self.attn_backend = _Backend.FLASH_ATTN
|
||
|
||
@property
|
||
def dtype(self) -> torch.dtype:
|
||
return self.patch_embed.proj.weight.dtype
|
||
|
||
@property
|
||
def device(self) -> torch.device:
|
||
return self.patch_embed.proj.weight.device
|
||
|
||
def rotary_pos_emb_thw(self, t, h, w):
|
||
hpos_ids = torch.arange(h).unsqueeze(1).expand(-1, w)
|
||
wpos_ids = torch.arange(w).unsqueeze(0).expand(h, -1)
|
||
hpos_ids = hpos_ids.reshape(
|
||
h // self.spatial_merge_size,
|
||
self.spatial_merge_size,
|
||
w // self.spatial_merge_size,
|
||
self.spatial_merge_size,
|
||
).permute(0, 2, 1, 3).flatten()
|
||
wpos_ids = wpos_ids.reshape(
|
||
h // self.spatial_merge_size,
|
||
self.spatial_merge_size,
|
||
w // self.spatial_merge_size,
|
||
self.spatial_merge_size,
|
||
).permute(0, 2, 1, 3).flatten()
|
||
pos_ids = torch.stack([hpos_ids, wpos_ids], dim=-1).repeat(t, 1)
|
||
max_size = max(h, w)
|
||
rotary_pos_emb_full = self.rotary_pos_emb(max_size)
|
||
rotary_pos_emb = rotary_pos_emb_full[pos_ids].flatten(1)
|
||
rotary_pos_emb = rotary_pos_emb.reshape(
|
||
rotary_pos_emb.shape[0] // self.spatial_merge_unit,
|
||
self.spatial_merge_unit, -1)
|
||
|
||
return rotary_pos_emb
|
||
|
||
def get_window_index_thw(self, grid_t, grid_h, grid_w):
|
||
vit_merger_window_size = (self.window_size //
|
||
self.spatial_merge_size // self.patch_size)
|
||
|
||
llm_grid_h = grid_h // self.spatial_merge_size
|
||
llm_grid_w = grid_w // self.spatial_merge_size
|
||
index = torch.arange(grid_t * llm_grid_h * llm_grid_w).reshape(
|
||
grid_t, llm_grid_h, llm_grid_w)
|
||
pad_h = vit_merger_window_size - llm_grid_h % vit_merger_window_size
|
||
pad_w = vit_merger_window_size - llm_grid_w % vit_merger_window_size
|
||
num_windows_h = (llm_grid_h + pad_h) // vit_merger_window_size
|
||
num_windows_w = (llm_grid_w + pad_w) // vit_merger_window_size
|
||
index_padded = F.pad(index, (0, pad_w, 0, pad_h), 'constant', -100)
|
||
index_padded = index_padded.reshape(grid_t, num_windows_h,
|
||
vit_merger_window_size,
|
||
num_windows_w,
|
||
vit_merger_window_size)
|
||
index_padded = index_padded.permute(0, 1, 3, 2, 4).reshape(
|
||
grid_t, num_windows_h * num_windows_w, vit_merger_window_size,
|
||
vit_merger_window_size)
|
||
seqlens = (index_padded != -100).sum([2, 3]).reshape(-1)
|
||
index_padded = index_padded.reshape(-1)
|
||
index_new = index_padded[index_padded != -100]
|
||
cu_seqlens_tmp = seqlens.cumsum(0) * self.spatial_merge_unit
|
||
cu_seqlens_tmp = cu_seqlens_tmp.to(dtype=torch.int32)
|
||
cu_seqlens_tmp = torch.unique_consecutive(cu_seqlens_tmp)
|
||
|
||
return index_new, cu_seqlens_tmp
|
||
|
||
@lru_cache(maxsize=1024) # noqa: B019
|
||
def get_rope_by_thw(self, t, h, w):
|
||
window_index_thw, cu_seqlens_window_thw = self.get_window_index_thw(
|
||
t, h, w)
|
||
rotary_pos_emb_thw = self.rotary_pos_emb_thw(t, h, w)
|
||
rotary_pos_emb_thw = rotary_pos_emb_thw[window_index_thw, :, :]
|
||
rotary_pos_emb_thw = rotary_pos_emb_thw.flatten(start_dim=0, end_dim=1)
|
||
cu_seqlens_thw = torch.repeat_interleave(
|
||
torch.tensor([h * w], dtype=torch.int32), t)
|
||
return (rotary_pos_emb_thw, window_index_thw, cu_seqlens_window_thw,
|
||
cu_seqlens_thw)
|
||
|
||
def compute_attn_mask_seqlen(
|
||
self,
|
||
cu_seqlens: torch.Tensor,
|
||
) -> tuple[Optional[int], Optional[list[int]]]:
|
||
max_seqlen, seqlens = None, None
|
||
if self.attn_backend == _Backend.FLASH_ATTN:
|
||
max_seqlen = (cu_seqlens[1:] - cu_seqlens[:-1]).max().item()
|
||
elif self.attn_backend == _Backend.XFORMERS:
|
||
seqlens = (cu_seqlens[1:] - cu_seqlens[:-1]).tolist()
|
||
return max_seqlen, seqlens
|
||
|
||
def forward(
|
||
self,
|
||
x: torch.Tensor,
|
||
grid_thw: list[list[int]],
|
||
) -> torch.Tensor:
|
||
# patchify
|
||
seq_len, _ = x.size()
|
||
rotary_pos_emb = []
|
||
window_index: list = []
|
||
cu_window_seqlens: list = [torch.tensor([0], dtype=torch.int32)]
|
||
cu_seqlens: list = []
|
||
|
||
hidden_states = x.to(device=self.device, dtype=self.dtype)
|
||
hidden_states = self.patch_embed(hidden_states)
|
||
|
||
window_index_id = 0
|
||
cu_window_seqlens_last = 0
|
||
for t, h, w in grid_thw:
|
||
t, h, w = int(t), int(h), int(w)
|
||
llm_h = h // self.spatial_merge_size
|
||
llm_w = w // self.spatial_merge_size
|
||
|
||
(
|
||
rotary_pos_emb_thw,
|
||
window_index_thw,
|
||
cu_seqlens_window_thw,
|
||
cu_seqlens_thw,
|
||
) = self.get_rope_by_thw(t, h, w)
|
||
|
||
window_index.append(window_index_thw + window_index_id)
|
||
window_index_id += (t * llm_h * llm_w)
|
||
|
||
cu_seqlens_window_thw = (cu_seqlens_window_thw +
|
||
cu_window_seqlens_last)
|
||
cu_window_seqlens_last = cu_seqlens_window_thw[-1]
|
||
cu_window_seqlens.append(cu_seqlens_window_thw)
|
||
|
||
rotary_pos_emb.append(rotary_pos_emb_thw)
|
||
|
||
cu_seqlens.append(cu_seqlens_thw)
|
||
|
||
rotary_pos_emb = torch.cat(rotary_pos_emb)
|
||
window_index = torch.cat(window_index)
|
||
cu_window_seqlens = torch.cat(cu_window_seqlens)
|
||
cu_window_seqlens = torch.unique_consecutive(cu_window_seqlens)
|
||
cu_seqlens = torch.cat(cu_seqlens)
|
||
cu_seqlens = torch.cumsum(cu_seqlens, dim=0, dtype=torch.int32)
|
||
cu_seqlens = F.pad(cu_seqlens, (1, 0), "constant", 0)
|
||
|
||
# transformers
|
||
# pre-compute seqlens for window/full attn to reduce cuMemcpy operations
|
||
max_seqlen_full, seqlens_full = self.compute_attn_mask_seqlen(
|
||
cu_seqlens)
|
||
max_seqlen_window, seqlens_window = self.compute_attn_mask_seqlen(
|
||
cu_window_seqlens)
|
||
|
||
cu_seqlens = cu_seqlens.to(device=self.device, non_blocking=True)
|
||
cu_window_seqlens = cu_window_seqlens.to(device=self.device,
|
||
non_blocking=True)
|
||
rotary_pos_emb = rotary_pos_emb.to(device=self.device,
|
||
non_blocking=True)
|
||
window_index = window_index.to(device=hidden_states.device,
|
||
non_blocking=True)
|
||
|
||
hidden_states = hidden_states.reshape(
|
||
seq_len // self.spatial_merge_unit, self.spatial_merge_unit, -1)
|
||
hidden_states = hidden_states[window_index, :, :]
|
||
hidden_states = hidden_states.reshape(seq_len, -1)
|
||
|
||
hidden_states = hidden_states.unsqueeze(1)
|
||
|
||
for layer_num, blk in enumerate(self.blocks):
|
||
if layer_num in self.fullatt_block_indexes:
|
||
cu_seqlens_now = cu_seqlens
|
||
max_seqlen_now = max_seqlen_full
|
||
seqlens_now = seqlens_full
|
||
else:
|
||
cu_seqlens_now = cu_window_seqlens
|
||
max_seqlen_now = max_seqlen_window
|
||
seqlens_now = seqlens_window
|
||
|
||
hidden_states = blk(
|
||
hidden_states,
|
||
cu_seqlens=cu_seqlens_now,
|
||
rotary_pos_emb=rotary_pos_emb,
|
||
max_seqlen=max_seqlen_now,
|
||
seqlens=seqlens_now,
|
||
)
|
||
|
||
# For Qwen2.5-VL-3B, float16 will overflow at last block
|
||
# for long visual tokens sequences.
|
||
if hidden_states.dtype == torch.float16:
|
||
hidden_states = cast_overflow_tensors(hidden_states)
|
||
|
||
# adapter
|
||
hidden_states = self.merger(hidden_states)
|
||
reverse_indices = torch.argsort(window_index)
|
||
hidden_states = hidden_states[reverse_indices, :]
|
||
return hidden_states
|
||
|
||
def load_weights(self, weights: Iterable[tuple[str,
|
||
torch.Tensor]]) -> set[str]:
|
||
stacked_params_mapping = [
|
||
# (param_name, shard_name, shard_id)
|
||
("attn.qkv.", "attn.q.", "q"),
|
||
("attn.qkv.", "attn.k.", "k"),
|
||
("attn.qkv.", "attn.v.", "v"),
|
||
]
|
||
params_dict = dict(self.named_parameters(remove_duplicate=False))
|
||
loaded_params: set[str] = set()
|
||
|
||
for name, loaded_weight in weights:
|
||
for (param_name, weight_name, shard_id) in stacked_params_mapping:
|
||
if weight_name not in name:
|
||
continue
|
||
name = name.replace(weight_name, param_name)
|
||
|
||
param = params_dict[name]
|
||
weight_loader = param.weight_loader
|
||
weight_loader(param, loaded_weight, shard_id)
|
||
break
|
||
else:
|
||
param = params_dict[name]
|
||
weight_loader = getattr(param, "weight_loader",
|
||
default_weight_loader)
|
||
weight_loader(param, loaded_weight)
|
||
loaded_params.add(name)
|
||
return loaded_params
|
||
|
||
|
||
class Qwen2_5_VLProcessingInfo(Qwen2VLProcessingInfo):
|
||
|
||
def get_hf_config(self):
|
||
return self.ctx.get_hf_config(Qwen2_5_VLConfig)
|
||
|
||
def get_hf_processor(
|
||
self,
|
||
*,
|
||
min_pixels: Optional[int] = None,
|
||
max_pixels: Optional[int] = None,
|
||
size: Optional[dict[str, int]] = None,
|
||
fps: Optional[Union[float, list[float]]] = None,
|
||
**kwargs: object,
|
||
) -> Qwen2_5_VLProcessor:
|
||
if fps is not None:
|
||
kwargs["fps"] = fps
|
||
|
||
return self.ctx.get_hf_processor(
|
||
Qwen2_5_VLProcessor,
|
||
image_processor=self.get_image_processor(min_pixels=min_pixels,
|
||
max_pixels=max_pixels,
|
||
size=size,
|
||
use_fast=kwargs.get(
|
||
"use_fast", True)),
|
||
**kwargs,
|
||
)
|
||
|
||
|
||
class Qwen2_5_VLMultiModalProcessor(Qwen2VLMultiModalProcessor):
|
||
|
||
def _get_mm_fields_config(
|
||
self,
|
||
hf_inputs: BatchFeature,
|
||
hf_processor_mm_kwargs: Mapping[str, object],
|
||
) -> Mapping[str, MultiModalFieldConfig]:
|
||
return dict(
|
||
**super()._get_mm_fields_config(hf_inputs, hf_processor_mm_kwargs),
|
||
second_per_grid_ts=MultiModalFieldConfig.batched("video"),
|
||
)
|
||
|
||
|
||
@MULTIMODAL_REGISTRY.register_processor(
|
||
Qwen2_5_VLMultiModalProcessor,
|
||
info=Qwen2_5_VLProcessingInfo,
|
||
dummy_inputs=Qwen2_5_VLDummyInputsBuilder)
|
||
class Qwen2_5_VLForConditionalGeneration(nn.Module, SupportsMultiModal,
|
||
SupportsLoRA, SupportsPP,
|
||
SupportsQuant):
|
||
|
||
# To ensure correct weight loading and mapping.
|
||
hf_to_vllm_mapper = WeightsMapper(
|
||
orig_to_new_prefix={
|
||
# mapping for new names in checkpoint saved after transformers v4.52
|
||
"model.language_model.": "language_model.model.",
|
||
"model.visual.": "visual.",
|
||
# mapping for original checkpoint
|
||
"lm_head.": "language_model.lm_head.",
|
||
"model.": "language_model.model.",
|
||
})
|
||
|
||
@classmethod
|
||
def get_placeholder_str(cls, modality: str, i: int) -> Optional[str]:
|
||
if modality.startswith("image"):
|
||
return "<|vision_start|><|image_pad|><|vision_end|>"
|
||
if modality.startswith("video"):
|
||
return "<|vision_start|><|video_pad|><|vision_end|>"
|
||
|
||
raise ValueError("Only image or video modality is supported")
|
||
|
||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||
super().__init__()
|
||
config: Qwen2_5_VLConfig = vllm_config.model_config.hf_config
|
||
multimodal_config = vllm_config.model_config.multimodal_config
|
||
|
||
self.config = config
|
||
self.multimodal_config = multimodal_config
|
||
|
||
self.visual = Qwen2_5_VisionTransformer(
|
||
config.vision_config,
|
||
norm_eps=getattr(config, "rms_norm_eps", 1e-6),
|
||
quant_config=self._maybe_ignore_quant_config(self.quant_config),
|
||
prefix=maybe_prefix(prefix, "visual"),
|
||
)
|
||
|
||
self.language_model = init_vllm_registered_model(
|
||
vllm_config=vllm_config,
|
||
prefix=maybe_prefix(prefix, "language_model"),
|
||
architectures=["Qwen2ForCausalLM"],
|
||
)
|
||
|
||
self.make_empty_intermediate_tensors = (
|
||
self.language_model.make_empty_intermediate_tensors)
|
||
|
||
def _maybe_ignore_quant_config(self, config: Optional[QuantizationConfig]):
|
||
# GPTQ configs do not have a list of ignored modules, however AutoGPTQ
|
||
# seems to avoid vision encoder sections for some models.
|
||
return config
|
||
|
||
def _validate_and_reshape_mm_tensor(self, mm_input: object,
|
||
name: str) -> torch.Tensor:
|
||
if not isinstance(mm_input, (torch.Tensor, list)):
|
||
raise ValueError(f"Incorrect type of {name}. "
|
||
f"Got type: {type(mm_input)}")
|
||
if isinstance(mm_input, torch.Tensor):
|
||
if mm_input.ndim == 2:
|
||
return mm_input
|
||
if mm_input.ndim != 3:
|
||
raise ValueError(f"{name} should be 2D or batched 3D tensor. "
|
||
f"Got ndim: {mm_input.ndim} "
|
||
f"(shape={mm_input.shape})")
|
||
return torch.concat(list(mm_input))
|
||
else:
|
||
return torch.concat(mm_input)
|
||
|
||
def _parse_and_validate_image_input(
|
||
self, **kwargs: object) -> Optional[Qwen2_5_VLImageInputs]:
|
||
pixel_values = kwargs.pop("pixel_values", None)
|
||
image_embeds = kwargs.pop("image_embeds", None)
|
||
image_grid_thw = kwargs.pop("image_grid_thw", None)
|
||
|
||
if pixel_values is None and image_embeds is None:
|
||
return None
|
||
|
||
if pixel_values is not None:
|
||
pixel_values = self._validate_and_reshape_mm_tensor(
|
||
pixel_values, "image pixel values")
|
||
image_grid_thw = self._validate_and_reshape_mm_tensor(
|
||
image_grid_thw, "image grid_thw")
|
||
|
||
if not isinstance(pixel_values, (torch.Tensor, list)):
|
||
raise ValueError("Incorrect type of image pixel values. "
|
||
f"Got type: {type(pixel_values)}")
|
||
|
||
return Qwen2_5_VLImagePixelInputs(type="pixel_values",
|
||
pixel_values=pixel_values,
|
||
image_grid_thw=image_grid_thw)
|
||
|
||
if image_embeds is not None:
|
||
image_embeds = self._validate_and_reshape_mm_tensor(
|
||
image_embeds, "image embeds")
|
||
image_grid_thw = self._validate_and_reshape_mm_tensor(
|
||
image_grid_thw, "image grid_thw")
|
||
|
||
if not isinstance(image_embeds, torch.Tensor):
|
||
raise ValueError("Incorrect type of image embeddings. "
|
||
f"Got type: {type(image_embeds)}")
|
||
return Qwen2_5_VLImageEmbeddingInputs(
|
||
type="image_embeds",
|
||
image_embeds=image_embeds,
|
||
image_grid_thw=image_grid_thw)
|
||
|
||
def _parse_and_validate_video_input(
|
||
self, **kwargs: object) -> Optional[Qwen2_5_VLVideoInputs]:
|
||
pixel_values_videos = kwargs.pop("pixel_values_videos", None)
|
||
video_embeds = kwargs.pop("video_embeds", None)
|
||
video_grid_thw = kwargs.pop("video_grid_thw", None)
|
||
second_per_grid_ts = kwargs.pop("second_per_grid_ts", None)
|
||
|
||
if pixel_values_videos is None and video_embeds is None:
|
||
return None
|
||
|
||
if pixel_values_videos is not None:
|
||
pixel_values_videos = self._validate_and_reshape_mm_tensor(
|
||
pixel_values_videos, "video pixel values")
|
||
video_grid_thw = self._validate_and_reshape_mm_tensor(
|
||
video_grid_thw, "video grid_thw")
|
||
|
||
return Qwen2_5_VLVideoPixelInputs(
|
||
type="pixel_values_videos",
|
||
pixel_values_videos=pixel_values_videos,
|
||
video_grid_thw=video_grid_thw,
|
||
second_per_grid_ts=second_per_grid_ts,
|
||
)
|
||
|
||
if video_embeds is not None:
|
||
video_embeds = self._validate_and_reshape_mm_tensor(
|
||
video_embeds, "video embeds")
|
||
video_grid_thw = self._validate_and_reshape_mm_tensor(
|
||
video_grid_thw, "video grid_thw")
|
||
|
||
if not isinstance(video_embeds, torch.Tensor):
|
||
raise ValueError("Incorrect type of video embeddings. "
|
||
f"Got type: {type(video_embeds)}")
|
||
return Qwen2_5_VLVideoEmbeddingInputs(
|
||
type="video_embeds",
|
||
video_embeds=video_embeds,
|
||
video_grid_thw=video_grid_thw)
|
||
|
||
def _process_image_input(
|
||
self,
|
||
image_input: Qwen2_5_VLImageInputs) -> tuple[torch.Tensor, ...]:
|
||
"""
|
||
处理图像输入，返回每个图像项的张量。
|
||
如果输入是图像嵌入，则返回图像嵌入；否则返回经过视觉模型处理后的张量。
|
||
|
||
Args:
|
||
image_input (Qwen2_5_VLImageInputs): 包含图像信息的字典，其中包括以下键值对：
|
||
- type (str, optional): 图像类型，可选值为"image_embeds"或者None（默认），表示使用图像嵌入；
|
||
None时，需要提供"pixel_values"和"image_grid_thw"键值对。
|
||
- pixel_values (torch.Tensor, optional): 图像像素值，shape为(batch_size, num_channels, height, width)，
|
||
dtype为float32，optional，只有当type为None时才需要提供。
|
||
- image_grid_thw (torch.Tensor, optional): 图像网格大小，shape为(batch_size, 2)，dtype为int64，
|
||
optional，只有当type为None时才需要提供。
|
||
|
||
Returns:
|
||
tuple (torch.Tensor, ...): 一个元组，包含每个图像项的张量，张量数量等于图像网格大小的纵向乘积。
|
||
"""
|
||
|
||
grid_thw = image_input["image_grid_thw"]
|
||
assert grid_thw.ndim == 2
|
||
grid_thw_list = grid_thw.tolist()
|
||
|
||
if image_input["type"] == "image_embeds":
|
||
image_embeds = image_input["image_embeds"]
|
||
else:
|
||
pixel_values = image_input["pixel_values"]
|
||
image_embeds = self.visual(pixel_values, grid_thw=grid_thw_list)
|
||
|
||
# Split concatenated embeddings for each image item.
|
||
merge_size = self.visual.spatial_merge_size
|
||
sizes = (grid_thw[:, 0] * grid_thw[:, 1] * grid_thw[:, 2]) // (merge_size * merge_size)
|
||
|
||
return image_embeds.split(sizes.tolist())
|
||
|
||
def _process_video_input(
|
||
self,
|
||
video_input: Qwen2_5_VLVideoInputs) -> tuple[torch.Tensor, ...]:
|
||
"""
|
||
处理视频输入，返回每个视频项的张量。
|
||
如果是视频嵌入，则直接返回；否则，使用视觉模型提取视频嵌入。
|
||
|
||
Args:
|
||
video_input (Qwen2_5_VLVideoInputs): 包含视频数据的字典，包括：
|
||
- type (str): 类型，可选值为"video_embeds"或"pixel_values_videos"。
|
||
- video_grid_thw (torch.Tensor): 视频网格大小，形状为(N, 2)，其中N是视频项数，第一维度表示视频项索引，最后两个维度分别代表高和宽。
|
||
- video_embeds (Optional[torch.Tensor]): 视频嵌入，仅当type为"video_embeds"时有效。
|
||
- pixel_values_videos (Optional[torch.Tensor]): 像素值视频，仅当type为"pixel_values_videos"时有效。
|
||
|
||
Returns:
|
||
tuple[torch.Tensor, ...]: 元组，包含每个视频项的张量，长度为N。
|
||
"""
|
||
|
||
grid_thw = video_input["video_grid_thw"]
|
||
assert grid_thw.ndim == 2
|
||
grid_thw_list = grid_thw.tolist()
|
||
|
||
if video_input["type"] == "video_embeds":
|
||
video_embeds = video_input["video_embeds"]
|
||
else:
|
||
pixel_values_videos = video_input["pixel_values_videos"]
|
||
video_embeds = self.visual(pixel_values_videos,
|
||
grid_thw=grid_thw_list)
|
||
|
||
# Split concatenated embeddings for each video item.
|
||
merge_size = self.visual.spatial_merge_size
|
||
sizes = grid_thw.prod(grid_thw.dim() - 1) // merge_size // merge_size
|
||
|
||
return video_embeds.split(sizes.tolist())
|
||
|
||
def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict:
|
||
mm_input_by_modality = {}
|
||
|
||
# Preserve the order of modalities if there are multiple of them
|
||
# from the order of kwargs.
|
||
for input_key in kwargs:
|
||
if input_key in ("pixel_values", "image_embeds"
|
||
) and "image" not in mm_input_by_modality:
|
||
mm_input_by_modality[
|
||
"image"] = self._parse_and_validate_image_input(**kwargs)
|
||
if input_key in ("pixel_values_videos", "video_embeds"
|
||
) and "video" not in mm_input_by_modality:
|
||
mm_input_by_modality[
|
||
"video"] = self._parse_and_validate_video_input(**kwargs)
|
||
return mm_input_by_modality
|
||
|
||
def get_language_model(self) -> torch.nn.Module:
|
||
return self.language_model
|
||
|
||
def get_multimodal_embeddings(self,
|
||
**kwargs: object) -> MultiModalEmbeddings:
|
||
|
||
mm_input_by_modality = self._parse_and_validate_multimodal_inputs(
|
||
**kwargs)
|
||
if not mm_input_by_modality:
|
||
return []
|
||
|
||
# The result multimodal_embeddings is tuple of tensors, with each
|
||
# tensor correspoending to a multimodal data item (image or video).
|
||
multimodal_embeddings: tuple[torch.Tensor, ...] = ()
|
||
|
||
# NOTE: It is important to iterate over the keys in this dictionary
|
||
# to preserve the order of the modalities.
|
||
for modality in mm_input_by_modality:
|
||
multimodal_input = mm_input_by_modality[modality]
|
||
if modality == "image":
|
||
vision_embeddings = self._process_image_input(multimodal_input)
|
||
multimodal_embeddings += vision_embeddings
|
||
if modality == "video":
|
||
video_embeddings = self._process_video_input(multimodal_input)
|
||
multimodal_embeddings += video_embeddings
|
||
return multimodal_embeddings
|
||
|
||
def get_input_embeddings(
|
||
self,
|
||
input_ids: torch.Tensor,
|
||
multimodal_embeddings: Optional[MultiModalEmbeddings] = None,
|
||
) -> torch.Tensor:
|
||
inputs_embeds = self.language_model.get_input_embeddings(input_ids)
|
||
if multimodal_embeddings is not None \
|
||
and len(multimodal_embeddings) != 0:
|
||
inputs_embeds = merge_multimodal_embeddings(
|
||
input_ids, inputs_embeds, multimodal_embeddings,
|
||
[self.config.image_token_id, self.config.video_token_id])
|
||
return inputs_embeds
|
||
|
||
def get_input_embeddings_v0(
|
||
self,
|
||
input_ids: torch.Tensor,
|
||
image_input: Optional[Qwen2_5_VLImageInputs] = None,
|
||
video_input: Optional[Qwen2_5_VLVideoInputs] = None,
|
||
) -> torch.Tensor:
|
||
inputs_embeds = self.get_input_embeddings(input_ids)
|
||
if image_input is not None:
|
||
image_embeds = self._process_image_input(image_input)
|
||
inputs_embeds = merge_multimodal_embeddings(
|
||
input_ids,
|
||
inputs_embeds,
|
||
image_embeds,
|
||
placeholder_token_id=self.config.image_token_id,
|
||
)
|
||
|
||
if video_input is not None:
|
||
video_embeds = self._process_video_input(video_input)
|
||
inputs_embeds = merge_multimodal_embeddings(
|
||
input_ids,
|
||
inputs_embeds,
|
||
video_embeds,
|
||
placeholder_token_id=self.config.video_token_id,
|
||
)
|
||
return inputs_embeds
|
||
|
||
def forward(
|
||
self,
|
||
input_ids: torch.Tensor,
|
||
positions: torch.Tensor,
|
||
intermediate_tensors: Optional[IntermediateTensors] = None,
|
||
inputs_embeds: Optional[torch.Tensor] = None,
|
||
kv_caches: list[torch.Tensor]= None,
|
||
**kwargs: object,
|
||
) -> Union[torch.Tensor, IntermediateTensors]:
|
||
"""Run forward pass for Qwen2.5-VL.
|
||
|
||
Args:
|
||
input_ids: Flattened (concatenated) input_ids corresponding to a
|
||
batch.
|
||
positions: Flattened (concatenated) position ids corresponding to a
|
||
batch.
|
||
**NOTE**: If mrope is enabled (default setting for Qwen2.5-VL
|
||
opensource models), the shape will be `(3, seq_len)`,
|
||
otherwise it will be `(seq_len,).
|
||
pixel_values: Pixel values to be fed to a model.
|
||
`None` if no images are passed.
|
||
image_grid_thw: Tensor `(n_images, 3)` of image 3D grid in LLM.
|
||
`None` if no images are passed.
|
||
pixel_values_videos: Pixel values of videos to be fed to a model.
|
||
`None` if no videos are passed.
|
||
video_grid_thw: Tensor `(n_videos, 3)` of video 3D grid in LLM.
|
||
`None` if no videos are passed.
|
||
second_per_grid_ts: Tensor `(num_videos)` of video time interval (
|
||
in seconds) for each grid along the temporal dimension in the
|
||
3D position IDs. `None` if no videos are passed.
|
||
"""
|
||
|
||
if intermediate_tensors is not None:
|
||
inputs_embeds = None
|
||
|
||
# NOTE: In v1, inputs_embeds is always generated at model runner from
|
||
# `get_multimodal_embeddings` and `get_input_embeddings`, this
|
||
# condition is only for v0 compatibility.
|
||
elif inputs_embeds is None:
|
||
image_input = self._parse_and_validate_image_input(**kwargs)
|
||
video_input = self._parse_and_validate_video_input(**kwargs)
|
||
|
||
if image_input is None and video_input is None:
|
||
inputs_embeds = None
|
||
else:
|
||
if uses_mrope(self.config):
|
||
assert positions.ndim == 2 and positions.size(0) == 3, (
|
||
"multimodal section rotary embedding requires "
|
||
f"(3, seq_len) positions, but got {positions.size()}")
|
||
inputs_embeds = self.get_input_embeddings_v0(
|
||
input_ids,
|
||
image_input=image_input,
|
||
video_input=video_input)
|
||
input_ids = None
|
||
|
||
hidden_states = self.language_model.model(
|
||
input_ids=input_ids,
|
||
positions=positions,
|
||
intermediate_tensors=intermediate_tensors,
|
||
inputs_embeds=inputs_embeds,
|
||
)
|
||
return hidden_states
|
||
|
||
def compute_logits(
|
||
self,
|
||
hidden_states: torch.Tensor,
|
||
# sampling_metadata: SamplingMetadata,
|
||
) -> Optional[torch.Tensor]:
|
||
return self.language_model.compute_logits(hidden_states)
|
||
# sampling_metadata)
|
||
|
||
def load_weights(self, weights: Iterable[tuple[str,
|
||
torch.Tensor]]) -> set[str]:
|
||
|
||
loader = AutoWeightsLoader(self)
|
||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||
|
||
def get_mm_mapping(self) -> MultiModelKeys:
|
||
"""
|
||
Get the module prefix in multimodal models
|
||
"""
|
||
return MultiModelKeys.from_string_field(
|
||
language_model="language_model",
|
||
connector="visual.merger.",
|
||
tower_model="visual.",
|
||
) |