EngineX-Hygon/sglang
5
0
Fork 0
Code Issues Pull Requests Actions 7 Projects Releases Wiki Activity

ci: reduce and refactor vlm ut and combine test files (#11062)

Browse Source
This commit is contained in:
Mick
2025-10-17 23:24:50 +08:00
committed by GitHub
parent d88ac9bc9a
commit 3e4c7da2f5
6 changed files with 85 additions and 555 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
python/sglang/bench_one_batch_server.py
View File

@@ -137,12 +137,10 @@ def generate_markdown_report(trace_dir, results: List["BenchmarkResult"]) -> str
# all results should share the same isl & osl # all results should share the same isl & osl
for result in results: for result in results:
base_url = os.getenv( base_url = os.getenv("TRACE_BASE_URL", "").rstrip("/")
"TRACE_BASE_URL", "https://github.com/sgl-project/ci-data/traces"
).rstrip("/")
relay_base = os.getenv( relay_base = os.getenv(
"PERFETTO_RELAY_URL", "PERFETTO_RELAY_URL",
"https://docs.sglang.ai/ci-data/pages/perfetto_relay.html", "",
).rstrip("/") ).rstrip("/")
summary += result.to_markdown_row(trace_dir, base_url, relay_base) summary += result.to_markdown_row(trace_dir, base_url, relay_base)

2
python/sglang/srt/models/mllama.py
View File

@@ -901,7 +901,7 @@ class MllamaForConditionalGeneration(nn.Module):
img = pixel_values[0, j] img = pixel_values[0, j]
num_tiles = img.shape[0] num_tiles = img.shape[0]
batched_images[i, j, :num_tiles] = img batched_images[i, j, :num_tiles] = img
batched_ar_ids[i, j] = mm_input.mm_items[0].aspect_ratio_id[0, j] batched_ar_ids[i, j] = mm_input.mm_items[0].aspect_ratio_ids[0, j]
batched_ar_mask[i, j, :num_tiles] = mm_input.mm_items[ batched_ar_mask[i, j, :num_tiles] = mm_input.mm_items[
0 0

3
test/srt/run_suite.py
View File

@@ -131,8 +131,7 @@ suites = {
TestFile("test_triton_sliding_window.py", 250), TestFile("test_triton_sliding_window.py", 250),
TestFile("test_utils_update_weights.py", 48), TestFile("test_utils_update_weights.py", 48),
TestFile("test_vision_chunked_prefill.py", 175), TestFile("test_vision_chunked_prefill.py", 175),
TestFile("test_vision_openai_server_a.py", 724), TestFile("test_vision_openai_server_a.py", 608),
TestFile("test_vision_openai_server_b.py", 446),
TestFile("test_vlm_input_format.py", 300), TestFile("test_vlm_input_format.py", 300),
], ],
"per-commit-2-gpu": [ "per-commit-2-gpu": [

328
test/srt/test_vision_openai_server_a.py
View File

@@ -8,91 +8,34 @@ import unittest
from test_vision_openai_server_common import * from test_vision_openai_server_common import *
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, class TestLlavaServer(ImageOpenAITestMixin):
popen_launch_server, model = "lmms-lab/llava-onevision-qwen2-0.5b-ov"
)
class TestLlava(ImageOpenAITestMixin): class TestQwen25VLServer(ImageOpenAITestMixin, VideoOpenAITestMixin):
@classmethod model = "Qwen/Qwen2.5-VL-7B-Instruct"
def setUpClass(cls): extra_args = [
cls.model = "lmms-lab/llava-onevision-qwen2-0.5b-ov" "--cuda-graph-max-bs=4",
cls.base_url = DEFAULT_URL_FOR_TEST ]
cls.api_key = "sk-123456"
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
api_key=cls.api_key,
)
cls.base_url += "/v1"
class TestQwen2VLServer(ImageOpenAITestMixin, VideoOpenAITestMixin):
@classmethod
def setUpClass(cls):
cls.model = "Qwen/Qwen2-VL-7B-Instruct"
cls.base_url = DEFAULT_URL_FOR_TEST
cls.api_key = "sk-123456"
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
api_key=cls.api_key,
other_args=[
"--mem-fraction-static",
"0.35",
"--cuda-graph-max-bs",
"4",
],
)
cls.base_url += "/v1"
class TestQwen3VLServer(ImageOpenAITestMixin, VideoOpenAITestMixin): class TestQwen3VLServer(ImageOpenAITestMixin, VideoOpenAITestMixin):
@classmethod model = "Qwen/Qwen3-VL-30B-A3B-Instruct"
def setUpClass(cls): extra_args = ["--cuda-graph-max-bs=4"]
cls.model = "Qwen/Qwen3-VL-30B-A3B-Instruct"
cls.base_url = DEFAULT_URL_FOR_TEST
cls.api_key = "sk-123456"
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
api_key=cls.api_key,
other_args=[
"--mem-fraction-static",
"0.80",
"--cuda-graph-max-bs",
"4",
],
)
cls.base_url += "/v1"
class TestQwen2_5_VLServer(ImageOpenAITestMixin, VideoOpenAITestMixin): class TestQwen3OmniServer(OmniOpenAITestMixin):
@classmethod model = "Qwen/Qwen3-Omni-30B-A3B-Instruct"
def setUpClass(cls): extra_args = [ # workaround to fit into H100
cls.model = "Qwen/Qwen2.5-VL-7B-Instruct" "--mem-fraction-static=0.90",
cls.base_url = DEFAULT_URL_FOR_TEST "--disable-cuda-graph",
cls.api_key = "sk-123456" "--disable-fast-image-processor",
cls.process = popen_launch_server( "--grammar-backend=none",
cls.model, ]
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
api_key=cls.api_key,
other_args=[
"--mem-fraction-static",
"0.35",
"--cuda-graph-max-bs",
"4",
],
)
cls.base_url += "/v1"
class TestVLMContextLengthIssue(CustomTestCase): class TestQwen2VLContextLengthServer(CustomTestCase):
@classmethod @classmethod
def setUpClass(cls): def setUpClass(cls):
cls.model = "Qwen/Qwen2-VL-7B-Instruct" cls.model = "Qwen/Qwen2-VL-7B-Instruct"
@@ -106,7 +49,6 @@ class TestVLMContextLengthIssue(CustomTestCase):
other_args=[ other_args=[
"--context-length", "--context-length",
"300", "300",
"--mem-fraction-static=0.75",
"--cuda-graph-max-bs", "--cuda-graph-max-bs",
"4", "4",
], ],
@@ -149,210 +91,62 @@ class TestVLMContextLengthIssue(CustomTestCase):
) )
# Note(Xinyuan): mllama is not stable for now, skip for CI # flaky
# class TestMllamaServer(TestOpenAIVisionServer): # class TestMllamaServer(ImageOpenAITestMixin):
# @classmethod # model = "meta-llama/Llama-3.2-11B-Vision-Instruct"
# def setUpClass(cls):
# cls.model = "meta-llama/Llama-3.2-11B-Vision-Instruct"
# cls.base_url = DEFAULT_URL_FOR_TEST
# cls.api_key = "sk-123456"
# cls.process = popen_launch_server(
# cls.model,
# cls.base_url,
# timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
# api_key=cls.api_key,
# )
# cls.base_url += "/v1"
class TestMinicpmvServer(ImageOpenAITestMixin): class TestInternVL25Server(ImageOpenAITestMixin):
@classmethod model = "OpenGVLab/InternVL2_5-2B"
def setUpClass(cls): extra_args = [
cls.model = "openbmb/MiniCPM-V-2_6" "--cuda-graph-max-bs=4",
cls.base_url = DEFAULT_URL_FOR_TEST ]
cls.api_key = "sk-123456"
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=[
"--trust-remote-code",
"--mem-fraction-static",
"0.35",
"--cuda-graph-max-bs",
"4",
],
)
cls.base_url += "/v1"
class TestMinicpmv4Server(ImageOpenAITestMixin): class TestMiniCPMV4Server(ImageOpenAITestMixin):
@classmethod model = "openbmb/MiniCPM-V-4"
def setUpClass(cls): extra_args = [
cls.model = "openbmb/MiniCPM-V-4" "--cuda-graph-max-bs=4",
cls.base_url = DEFAULT_URL_FOR_TEST ]
cls.api_key = "sk-123456"
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=[
"--trust-remote-code",
"--mem-fraction-static",
"0.35",
"--cuda-graph-max-bs",
"4",
],
)
cls.base_url += "/v1"
class TestInternVL2_5Server(ImageOpenAITestMixin): class TestMiniCPMo26Server(ImageOpenAITestMixin, AudioOpenAITestMixin):
@classmethod model = "openbmb/MiniCPM-o-2_6"
def setUpClass(cls): extra_args = [
cls.model = "OpenGVLab/InternVL2_5-2B" "--cuda-graph-max-bs=4",
cls.base_url = DEFAULT_URL_FOR_TEST ]
cls.api_key = "sk-123456"
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=[
"--trust-remote-code",
"--cuda-graph-max-bs",
"4",
],
)
cls.base_url += "/v1"
class TestMinicpmo2_6Server(ImageOpenAITestMixin, AudioOpenAITestMixin): class TestGemma3itServer(ImageOpenAITestMixin):
@classmethod model = "google/gemma-3-4b-it"
def setUpClass(cls): extra_args = [
cls.model = "openbmb/MiniCPM-o-2_6" "--cuda-graph-max-bs=4",
cls.base_url = DEFAULT_URL_FOR_TEST ]
cls.api_key = "sk-123456"
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=[
"--trust-remote-code",
"--mem-fraction-static",
"0.65",
"--cuda-graph-max-bs",
"4",
],
)
cls.base_url += "/v1"
class TestMimoVLServer(ImageOpenAITestMixin): class TestKimiVLServer(ImageOpenAITestMixin):
@classmethod model = "moonshotai/Kimi-VL-A3B-Instruct"
def setUpClass(cls): extra_args = [
cls.model = "XiaomiMiMo/MiMo-VL-7B-RL" "--context-length=8192",
cls.base_url = DEFAULT_URL_FOR_TEST "--dtype=bfloat16",
cls.api_key = "sk-123456" ]
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
api_key=cls.api_key,
other_args=[
"--trust-remote-code",
"--mem-fraction-static",
"0.6",
"--cuda-graph-max-bs",
"4",
],
)
cls.base_url += "/v1"
def test_video_images_chat_completion(self):
class TestVILAServer(ImageOpenAITestMixin): # model context length exceeded
@classmethod
def setUpClass(cls):
cls.model = "Efficient-Large-Model/NVILA-Lite-2B-hf-0626"
cls.base_url = DEFAULT_URL_FOR_TEST
cls.api_key = "sk-123456"
cls.revision = "6bde1de5964b40e61c802b375fff419edc867506"
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
api_key=cls.api_key,
other_args=[
"--trust-remote-code",
"--context-length=65536",
f"--revision={cls.revision}",
"--cuda-graph-max-bs",
"4",
],
)
cls.base_url += "/v1"
class TestPhi4MMServer(ImageOpenAITestMixin, AudioOpenAITestMixin):
@classmethod
def setUpClass(cls):
# Manually download LoRA adapter_config.json as it's not downloaded by the model loader by default.
from huggingface_hub import constants, snapshot_download
snapshot_download(
"microsoft/Phi-4-multimodal-instruct",
allow_patterns=["**/adapter_config.json"],
)
cls.model = "microsoft/Phi-4-multimodal-instruct"
cls.base_url = DEFAULT_URL_FOR_TEST
cls.api_key = "sk-123456"
revision = "33e62acdd07cd7d6635badd529aa0a3467bb9c6a"
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=[
"--trust-remote-code",
"--mem-fraction-static",
"0.70",
"--disable-radix-cache",
"--max-loras-per-batch",
"2",
"--revision",
revision,
"--lora-paths",
f"vision={constants.HF_HUB_CACHE}/models--microsoft--Phi-4-multimodal-instruct/snapshots/{revision}/vision-lora",
f"speech={constants.HF_HUB_CACHE}/models--microsoft--Phi-4-multimodal-instruct/snapshots/{revision}/speech-lora",
"--cuda-graph-max-bs",
"4",
],
)
cls.base_url += "/v1"
def get_vision_request_kwargs(self):
return {
"extra_body": {
"lora_path": "vision",
"top_k": 1,
"top_p": 1.0,
}
}
def get_audio_request_kwargs(self):
return {
"extra_body": {
"lora_path": "speech",
"top_k": 1,
"top_p": 1.0,
}
}
# This _test_audio_ambient_completion test is way too complicated to pass for a small LLM
def test_audio_ambient_completion(self):
pass pass
class TestGLM41VServer(ImageOpenAITestMixin, VideoOpenAITestMixin):
model = "zai-org/GLM-4.1V-9B-Thinking"
extra_args = [
"--reasoning-parser=glm45",
]
class TestQwen2AudioServer(AudioOpenAITestMixin):
model = "Qwen/Qwen2-Audio-7B-Instruct"
if __name__ == "__main__": if __name__ == "__main__":
del ( del (
TestOpenAIMLLMServerBase, TestOpenAIMLLMServerBase,

275
test/srt/test_vision_openai_server_b.py
View File

@@ -1,275 +0,0 @@
import unittest
from test_vision_openai_server_common import *
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
popen_launch_server,
)
class TestPixtralServer(ImageOpenAITestMixin):
@classmethod
def setUpClass(cls):
cls.model = "mistral-community/pixtral-12b"
cls.base_url = DEFAULT_URL_FOR_TEST
cls.api_key = "sk-123456"
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=[
"--trust-remote-code",
"--mem-fraction-static",
"0.70",
"--cuda-graph-max-bs",
"4",
],
)
cls.base_url += "/v1"
class TestMistral3_1Server(ImageOpenAITestMixin):
@classmethod
def setUpClass(cls):
cls.model = "unsloth/Mistral-Small-3.1-24B-Instruct-2503"
cls.base_url = DEFAULT_URL_FOR_TEST
cls.api_key = "sk-123456"
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=[
"--trust-remote-code",
"--mem-fraction-static",
"0.75",
"--cuda-graph-max-bs",
"4",
],
)
cls.base_url += "/v1"
class TestDeepseekVL2Server(ImageOpenAITestMixin):
@classmethod
def setUpClass(cls):
cls.model = "deepseek-ai/deepseek-vl2-small"
cls.base_url = DEFAULT_URL_FOR_TEST
cls.api_key = "sk-123456"
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=[
"--trust-remote-code",
"--context-length",
"4096",
"--cuda-graph-max-bs",
"4",
],
)
cls.base_url += "/v1"
class TestJanusProServer(ImageOpenAITestMixin):
@classmethod
def setUpClass(cls):
cls.model = "deepseek-ai/Janus-Pro-7B"
cls.base_url = DEFAULT_URL_FOR_TEST
cls.api_key = "sk-123456"
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=[
"--trust-remote-code",
"--mem-fraction-static",
"0.35",
"--cuda-graph-max-bs",
"4",
],
)
cls.base_url += "/v1"
def test_video_images_chat_completion(self):
pass
## Skip for ci test
# class TestLlama4Server(TestOpenAIVisionServer):
# @classmethod
# def setUpClass(cls):
# cls.model = "meta-llama/Llama-4-Scout-17B-16E-Instruct"
# cls.base_url = DEFAULT_URL_FOR_TEST
# cls.api_key = "sk-123456"
# cls.process = popen_launch_server(
# cls.model,
# cls.base_url,
# timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
# other_args=[
# "--chat-template",
# "llama-4",
# "--mem-fraction-static",
# "0.8",
# "--tp-size=8",
# "--context-length=8192",
# "--mm-attention-backend",
# "fa3",
# "--cuda-graph-max-bs",
# "4",
# ],
# )
# cls.base_url += "/v1"
class TestGemma3itServer(ImageOpenAITestMixin):
@classmethod
def setUpClass(cls):
cls.model = "google/gemma-3-4b-it"
cls.base_url = DEFAULT_URL_FOR_TEST
cls.api_key = "sk-123456"
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=[
"--trust-remote-code",
"--mem-fraction-static",
"0.70",
"--enable-multimodal",
"--cuda-graph-max-bs",
"4",
],
)
cls.base_url += "/v1"
class TestGemma3nServer(ImageOpenAITestMixin, AudioOpenAITestMixin):
@classmethod
def setUpClass(cls):
cls.model = "google/gemma-3n-E4B-it"
cls.base_url = DEFAULT_URL_FOR_TEST
cls.api_key = "sk-123456"
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=[
"--trust-remote-code",
"--mem-fraction-static",
"0.70",
"--cuda-graph-max-bs",
"4",
],
)
cls.base_url += "/v1"
# This _test_audio_ambient_completion test is way too complicated to pass for a small LLM
def test_audio_ambient_completion(self):
pass
def _test_mixed_image_audio_chat_completion(self):
self._test_mixed_image_audio_chat_completion()
class TestQwen2AudioServer(AudioOpenAITestMixin):
@classmethod
def setUpClass(cls):
cls.model = "Qwen/Qwen2-Audio-7B-Instruct"
cls.base_url = DEFAULT_URL_FOR_TEST
cls.api_key = "sk-123456"
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=[
"--trust-remote-code",
"--mem-fraction-static",
"0.70",
],
)
cls.base_url += "/v1"
# Temporarily skip Kimi-VL for CI test due to issue in transformers=4.57.0
# class TestKimiVLServer(ImageOpenAITestMixin):
# @classmethod
# def setUpClass(cls):
# cls.model = "moonshotai/Kimi-VL-A3B-Instruct"
# cls.base_url = DEFAULT_URL_FOR_TEST
# cls.api_key = "sk-123456"
# cls.process = popen_launch_server(
# cls.model,
# cls.base_url,
# timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
# other_args=[
# "--trust-remote-code",
# "--context-length",
# "4096",
# "--dtype",
# "bfloat16",
# "--cuda-graph-max-bs",
# "4",
# ],
# )
# cls.base_url += "/v1"
# def test_video_images_chat_completion(self):
# pass
class TestGLM41VServer(ImageOpenAITestMixin, VideoOpenAITestMixin):
@classmethod
def setUpClass(cls):
cls.model = "zai-org/GLM-4.1V-9B-Thinking"
cls.base_url = DEFAULT_URL_FOR_TEST
cls.api_key = "sk-123456"
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=[
"--trust-remote-code",
"--mem-fraction-static",
"0.68",
"--cuda-graph-max-bs",
"4",
"--reasoning-parser",
"glm45",
],
)
cls.base_url += "/v1"
class TestQwen3OmniServer(OmniOpenAITestMixin):
@classmethod
def setUpClass(cls):
cls.model = "Qwen/Qwen3-Omni-30B-A3B-Instruct"
cls.base_url = DEFAULT_URL_FOR_TEST
cls.api_key = "sk-123456"
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=[ # workaround to fit into H100
"--trust-remote-code",
"--mem-fraction-static",
"0.90",
"--disable-cuda-graph",
"--disable-fast-image-processor",
"--grammar-backend",
"none",
],
)
cls.base_url += "/v1"
if __name__ == "__main__":
del (
TestOpenAIMLLMServerBase,
ImageOpenAITestMixin,
VideoOpenAITestMixin,
AudioOpenAITestMixin,
OmniOpenAITestMixin,
)
unittest.main()

26
test/srt/test_vision_openai_server_common.py
View File

@@ -9,7 +9,12 @@ import requests
from PIL import Image from PIL import Image
from sglang.srt.utils import kill_process_tree from sglang.srt.utils import kill_process_tree
from sglang.test.test_utils import DEFAULT_URL_FOR_TEST, CustomTestCase from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
CustomTestCase,
popen_launch_server,
)
# image # image
IMAGE_MAN_IRONING_URL = "https://raw.githubusercontent.com/sgl-project/sgl-test-files/refs/heads/main/images/man_ironing_on_back_of_suv.png" IMAGE_MAN_IRONING_URL = "https://raw.githubusercontent.com/sgl-project/sgl-test-files/refs/heads/main/images/man_ironing_on_back_of_suv.png"
@@ -24,12 +29,21 @@ AUDIO_BIRD_SONG_URL = "https://raw.githubusercontent.com/sgl-project/sgl-test-fi
class TestOpenAIMLLMServerBase(CustomTestCase): class TestOpenAIMLLMServerBase(CustomTestCase):
model: str
extra_args: list = []
fixed_args: list = ["--trust-remote-code", "--enable-multimodal"]
@classmethod @classmethod
def setUpClass(cls): def setUpClass(cls):
cls.model = ""
cls.base_url = DEFAULT_URL_FOR_TEST cls.base_url = DEFAULT_URL_FOR_TEST
cls.api_key = "sk-123456" cls.api_key = "sk-123456"
cls.process = None cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
api_key=cls.api_key,
other_args=cls.extra_args + cls.fixed_args,
)
cls.base_url += "/v1" cls.base_url += "/v1"
@classmethod @classmethod
@@ -421,7 +435,7 @@ class ImageOpenAITestMixin(TestOpenAIMLLMServerBase):
or "device" in video_response or "device" in video_response
or "microphone" in video_response or "microphone" in video_response
), f""" ), f"""
====================== video_response ===================== ====================== video_images response =====================
{video_response} {video_response}
=========================================================== ===========================================================
should contain 'iPod' or 'device' or 'microphone' should contain 'iPod' or 'device' or 'microphone'
@@ -435,7 +449,7 @@ class ImageOpenAITestMixin(TestOpenAIMLLMServerBase):
or "Steve" in video_response or "Steve" in video_response
or "hand" in video_response or "hand" in video_response
), f""" ), f"""
====================== video_response ===================== ====================== video_images response =====================
{video_response} {video_response}
=========================================================== ===========================================================
should contain 'man' or 'person' or 'individual' or 'speaker' or 'presenter' or 'Steve' or 'hand' should contain 'man' or 'person' or 'individual' or 'speaker' or 'presenter' or 'Steve' or 'hand'
@@ -446,7 +460,7 @@ class ImageOpenAITestMixin(TestOpenAIMLLMServerBase):
or "display" in video_response or "display" in video_response
or "hold" in video_response or "hold" in video_response
), f""" ), f"""
====================== video_response ===================== ====================== video_images response =====================
{video_response} {video_response}
=========================================================== ===========================================================
should contain 'present' or 'examine' or 'display' or 'hold' should contain 'present' or 'examine' or 'display' or 'hold'