EngineX-Hygon/sglang
5
0
Fork 0
Code Issues Pull Requests Actions 7 Projects Releases Wiki Activity

[Refactor] simplify multimodal data processing (#8107)

Browse Source 
Signed-off-by: Xinyuan Tong <justinning0323@outlook.com>
This commit is contained in:
Xinyuan Tong
2025-07-20 21:43:09 -07:00
committed by GitHub
parent c9e8613c97
commit 8430bfe3e9
30 changed files with 297 additions and 421 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
test/srt/test_vision_openai_server_a.py
View File

@@ -116,22 +116,23 @@ class TestVLMContextLengthIssue(CustomTestCase):
)
class TestMllamaServer(TestOpenAIVisionServer):
@classmethod
def setUpClass(cls):
cls.model = "meta-llama/Llama-3.2-11B-Vision-Instruct"
cls.base_url = DEFAULT_URL_FOR_TEST
cls.api_key = "sk-123456"
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
api_key=cls.api_key,
)
cls.base_url += "/v1"
# Note(Xinyuan): mllama is not stable for now, skip for CI
# class TestMllamaServer(TestOpenAIVisionServer):
# @classmethod
# def setUpClass(cls):
# cls.model = "meta-llama/Llama-3.2-11B-Vision-Instruct"
# cls.base_url = DEFAULT_URL_FOR_TEST
# cls.api_key = "sk-123456"
# cls.process = popen_launch_server(
# cls.model,
# cls.base_url,
# timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
# api_key=cls.api_key,
# )
# cls.base_url += "/v1"
def test_video_chat_completion(self):
pass
# def test_video_chat_completion(self):
# pass
class TestMinicpmvServer(TestOpenAIVisionServer):

1
test/srt/test_vision_openai_server_b.py
View File

@@ -67,6 +67,7 @@ class TestDeepseekVL2Server(TestOpenAIVisionServer):
"--trust-remote-code",
"--context-length",
"4096",
"--disable-cuda-graph",
],
)
cls.base_url += "/v1"

22
test/srt/test_vision_openai_server_common.py
View File

@@ -308,19 +308,35 @@ class TestOpenAIVisionServer(CustomTestCase):
"iPod" in video_response
or "device" in video_response
or "microphone" in video_response
), video_response
), f"""
====================== video_response =====================
{video_response}
===========================================================
should contain 'iPod' or 'device' or 'microphone'
"""
assert (
"man" in video_response
or "person" in video_response
or "individual" in video_response
or "speaker" in video_response
), video_response
or "Steve" in video_response
), f"""
====================== video_response =====================
{video_response}
===========================================================
should contain 'man' or 'person' or 'individual' or 'speaker'
"""
assert (
"present" in video_response
or "examine" in video_response
or "display" in video_response
or "hold" in video_response
)
), f"""
====================== video_response =====================
{video_response}
===========================================================
should contain 'present' or 'examine' or 'display' or 'hold'
"""
assert "black" in video_response or "dark" in video_response
self.assertIsNotNone(video_response)
self.assertGreater(len(video_response), 0)

10
test/srt/test_vlm_input_format.py
View File

@@ -104,15 +104,15 @@ class VLMInputTestBase:
)
self.verify_response(output)
async def test_understands_precomputed_features(self):
async def test_understands_precomputed_embeddings(self):
req = self.get_completion_request()
processor_output = self.get_processor_output(req=req)
with torch.inference_mode():
precomputed_features = self.__class__.visual(processor_output)
precomputed_embeddings = self.__class__.visual(processor_output)
output = await self.engine.async_generate(
input_ids=processor_output["input_ids"][0].detach().cpu().tolist(),
image_data=[
self._precomputed_image_data(processor_output, precomputed_features)
self._precomputed_image_data(processor_output, precomputed_embeddings)
],
sampling_params=dict(temperature=0.0),
)
@@ -128,11 +128,11 @@ class VLMInputTestBase:
)
self.verify_response(output)
def _precomputed_image_data(self, processor_output, precomputed_features):
def _precomputed_image_data(self, processor_output, precomputed_embeddings):
"""This should not be overridden."""
return dict(
modality="IMAGE",
precomputed_features=precomputed_features,
precomputed_embeddings=precomputed_embeddings,
)
def _pixel_values_image_data(self, processor_output):