From bd70ce828cdb83e18fdd6ea6bc7b663961cb048a Mon Sep 17 00:00:00 2001
From: Li Wang <wangli858794774@gmail.com>
Date: Thu, 24 Apr 2025 17:12:12 +0800
Subject: [PATCH] [CI] Add qwen2.5-vl test (#643)

### What this PR does / why we need it?
Part of #499
Add qwen2.5-vl test on single npu, v1 engine is excluded because
qwen2.5-vl has some problems with v1 now, at the same time, this test
can also make #639 more credible

Signed-off-by: wangli <wangli858794774@gmail.com>
---
 tests/conftest.py                          |  7 +++++-
 tests/model_utils.py                       | 15 +++++++++++-
 tests/singlecard/test_offline_inference.py | 28 ++++++++++++++++++++++
 3 files changed, 48 insertions(+), 2 deletions(-)

diff --git a/tests/conftest.py b/tests/conftest.py
index 606ff83..78ffe8f 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -31,7 +31,7 @@ from vllm.outputs import RequestOutput
 from vllm.sampling_params import BeamSearchParams
 from vllm.utils import is_list_of
 
-from tests.model_utils import (TokensTextLogprobs,
+from tests.model_utils import (PROMPT_TEMPLATES, TokensTextLogprobs,
                                TokensTextLogprobsPromptLogprobs)
 # TODO: remove this part after the patch merged into vllm, if
 # we not explicitly patch here, some of them might be effectiveless
@@ -344,3 +344,8 @@ class VllmRunner:
 @pytest.fixture(scope="session")
 def vllm_runner():
     return VllmRunner
+
+
+@pytest.fixture(params=list(PROMPT_TEMPLATES.keys()))
+def prompt_template(request):
+    return PROMPT_TEMPLATES[request.param]
\ No newline at end of file
diff --git a/tests/model_utils.py b/tests/model_utils.py
index 1aedd1c..2ccc0d3 100644
--- a/tests/model_utils.py
+++ b/tests/model_utils.py
@@ -18,7 +18,7 @@
 #
 
 import warnings
-from typing import Dict, List, Optional, Sequence, Tuple, Union
+from typing import Callable, Dict, List, Optional, Sequence, Tuple, Union
 
 import torch
 from vllm.config import ModelConfig, TaskOption
@@ -301,3 +301,16 @@ def build_model_context(model_name: str,
         limit_mm_per_prompt=limit_mm_per_prompt,
     )
     return InputContext(model_config)
+
+
+def qwen_prompt(questions: List[str]) -> List[str]:
+    placeholder = "<|image_pad|>"
+    return [("<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n"
+             f"<|im_start|>user\n<|vision_start|>{placeholder}<|vision_end|>"
+             f"{q}<|im_end|>\n<|im_start|>assistant\n") for q in questions]
+
+
+# Map of prompt templates for different models.
+PROMPT_TEMPLATES: dict[str, Callable] = {
+    "qwen2.5vl": qwen_prompt,
+}
\ No newline at end of file
diff --git a/tests/singlecard/test_offline_inference.py b/tests/singlecard/test_offline_inference.py
index 7ccd9cf..5c10479 100644
--- a/tests/singlecard/test_offline_inference.py
+++ b/tests/singlecard/test_offline_inference.py
@@ -24,6 +24,7 @@ import os
 
 import pytest
 import vllm  # noqa: F401
+from vllm.assets.image import ImageAsset
 
 import vllm_ascend  # noqa: F401
 from tests.conftest import VllmRunner
@@ -32,6 +33,7 @@ MODELS = [
     "Qwen/Qwen2.5-0.5B-Instruct",
     "vllm-ascend/Qwen2.5-0.5B-Instruct-w8a8",
 ]
+MULTIMODALITY_MODELS = ["Qwen/Qwen2.5-VL-3B-Instruct"]
 os.environ["VLLM_USE_MODELSCOPE"] = "True"
 os.environ["PYTORCH_NPU_ALLOC_CONF"] = "max_split_size_mb:256"
 
@@ -55,6 +57,32 @@ def test_models(model: str, dtype: str, max_tokens: int) -> None:
         vllm_model.generate_greedy(example_prompts, max_tokens)
 
 
+@pytest.mark.parametrize("model", MULTIMODALITY_MODELS)
+@pytest.mark.skipif(os.getenv("VLLM_USE_V1") == "1",
+                    reason="qwen2.5_vl is not supported on v1")
+def test_multimodal(model, prompt_template, vllm_runner):
+    image = ImageAsset("cherry_blossom") \
+        .pil_image.convert("RGB")
+    img_questions = [
+        "What is the content of this image?",
+        "Describe the content of this image in detail.",
+        "What's in the image?",
+        "Where is this image taken?",
+    ]
+    images = [image] * len(img_questions)
+    prompts = prompt_template(img_questions)
+    with vllm_runner(model,
+                     max_model_len=4096,
+                     mm_processor_kwargs={
+                         "min_pixels": 28 * 28,
+                         "max_pixels": 1280 * 28 * 28,
+                         "fps": 1,
+                     }) as vllm_model:
+        vllm_model.generate_greedy(prompts=prompts,
+                                   images=images,
+                                   max_tokens=64)
+
+
 if __name__ == "__main__":
     import pytest
     pytest.main([__file__])