From f9842560cbd5bb7b1106b3f47982adb479a91774 Mon Sep 17 00:00:00 2001 From: tingfu Date: Sat, 8 Nov 2025 13:56:05 +0800 Subject: [PATCH] [0.11.0][Perf] Add padding vision tower for Qwen2_5_Omni (#4041) ### What this PR does / why we need it? This PR repalce the vision tower in Qwen2.5-Omni-Thinker model, Qwen2_5_VisionTransformer, with AscendQwen2_5_VisionTransformer, which use QKV padding for padding performance. - vLLM version: v0.11.0rc3 - vLLM main: https://github.com/vllm-project/vllm/commit/v0.11.0 Signed-off-by: Ting FU --- vllm_ascend/models/__init__.py | 4 ++ vllm_ascend/models/qwen2_5_omni_thinker.py | 54 ++++++++++++++++++++++ vllm_ascend/models/qwen2_5_vl.py | 8 ++++ 3 files changed, 66 insertions(+) create mode 100644 vllm_ascend/models/qwen2_5_omni_thinker.py diff --git a/vllm_ascend/models/__init__.py b/vllm_ascend/models/__init__.py index 2ebbdeb..7bd3e8b 100644 --- a/vllm_ascend/models/__init__.py +++ b/vllm_ascend/models/__init__.py @@ -23,6 +23,10 @@ def register_model(): "Qwen2_5_VLForConditionalGeneration", "vllm_ascend.models.qwen2_5_vl:AscendQwen2_5_VLForConditionalGeneration" ) + ModelRegistry.register_model( + "Qwen2_5OmniModel", + "vllm_ascend.models.qwen2_5_omni_thinker:AscendQwen2_5OmniThinkerForConditionalGeneration" + ) else: ModelRegistry.register_model( "Qwen2_5_VLForConditionalGeneration", diff --git a/vllm_ascend/models/qwen2_5_omni_thinker.py b/vllm_ascend/models/qwen2_5_omni_thinker.py new file mode 100644 index 0000000..ab43f1f --- /dev/null +++ b/vllm_ascend/models/qwen2_5_omni_thinker.py @@ -0,0 +1,54 @@ +# +# Copyright (c) 2025 Huawei Technologies Co., Ltd. All Rights Reserved. +# Adapted from vllm/model_executor/models/qwen2_5_vl.py +# Copyright 2023 The vLLM team. +# +# This file is a part of the vllm-ascend project. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from transformers.models.qwen2_5_omni.configuration_qwen2_5_omni import \ + Qwen2_5OmniThinkerConfig +from vllm.config import VllmConfig +from vllm.model_executor.models.qwen2_5_omni_thinker import ( + Qwen2_5OmniThinkerDummyInputsBuilder, + Qwen2_5OmniThinkerForConditionalGeneration, + Qwen2_5OmniThinkerMultiModalProcessor, Qwen2_5OmniThinkerProcessingInfo) +from vllm.model_executor.models.utils import maybe_prefix +from vllm.multimodal import MULTIMODAL_REGISTRY + +from vllm_ascend.models.qwen2_5_vl import AscendQwen2_5_VisionTransformer + + +@MULTIMODAL_REGISTRY.register_processor( + Qwen2_5OmniThinkerMultiModalProcessor, + info=Qwen2_5OmniThinkerProcessingInfo, + dummy_inputs=Qwen2_5OmniThinkerDummyInputsBuilder) +class AscendQwen2_5OmniThinkerForConditionalGeneration( + Qwen2_5OmniThinkerForConditionalGeneration): + + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): + + super().__init__(vllm_config=vllm_config, prefix=prefix) + config: Qwen2_5OmniThinkerConfig = vllm_config.model_config.hf_config.thinker_config + quant_config = vllm_config.quant_config + # The following code reuse AscendQwen2_5_VisionTransformer from Qwen2_5_VL, + # which does not import any model strcut difference. And will not impact + # the modeling files removing. + self.visual = AscendQwen2_5_VisionTransformer( + vision_config=config.vision_config, + norm_eps=getattr(config, "rms_norm_eps", 1e-6), + quant_config=quant_config, + prefix=maybe_prefix(prefix, "visual"), + ) diff --git a/vllm_ascend/models/qwen2_5_vl.py b/vllm_ascend/models/qwen2_5_vl.py index b19ed87..35ac58d 100644 --- a/vllm_ascend/models/qwen2_5_vl.py +++ b/vllm_ascend/models/qwen2_5_vl.py @@ -349,6 +349,9 @@ class AscendQwen2_5_VisionTransformer(Qwen2_5_VisionTransformer): ("qkv_proj", "q_proj", "q"), ("qkv_proj", "k_proj", "k"), ("qkv_proj", "v_proj", "v"), + ("attn.qkv.", "attn.q.", "q"), + ("attn.qkv.", "attn.k.", "k"), + ("attn.qkv.", "attn.v.", "v"), ("mlp.gate_up_proj.", "mlp.gate_proj.", 0), ("mlp.gate_up_proj.", "mlp.up_proj.", 1), ] @@ -363,6 +366,11 @@ class AscendQwen2_5_VisionTransformer(Qwen2_5_VisionTransformer): param = params_dict[name] weight_loader = param.weight_loader weight_loader(param, loaded_weight, shard_id) + if self.enable_pad and shard_id == "v": + if "attn.qkv.weight" in name: + param.data = self.pad_qkv_weight(param.data) + if "attn.qkv.bias" in name: + param.data = self.pad_qkv_bias(param.data) break else: param = params_dict[name]