EngineX-Hygon/sglang
5
0
Fork 0
Code Issues Pull Requests Actions 7 Projects Releases Wiki Activity

Fix chat template handling for OpenAI serving (#8635)

Browse Source 
Signed-off-by: Xinyuan Tong <justinning0323@outlook.com>
Signed-off-by: Xinyuan Tong <xinyuantong.cs@gmail.com>
This commit is contained in:
Xinyuan Tong
2025-07-31 21:49:45 -07:00
committed by GitHub
parent 20b5563eda
commit 7e831efee8
7 changed files with 83 additions and 156 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
test/srt/test_vision_openai_server_a.py
View File

@@ -31,6 +31,8 @@ class TestQwen2VLServer(TestOpenAIVisionServer):
other_args=[
"--mem-fraction-static",
"0.35",
"--cuda-graph-max-bs",
"4",
],
)
cls.base_url += "/v1"
@@ -53,6 +55,8 @@ class TestQwen2_5_VLServer(TestOpenAIVisionServer):
other_args=[
"--mem-fraction-static",
"0.35",
"--cuda-graph-max-bs",
"4",
],
)
cls.base_url += "/v1"
@@ -76,6 +80,8 @@ class TestVLMContextLengthIssue(CustomTestCase):
"--context-length",
"300",
"--mem-fraction-static=0.75",
"--cuda-graph-max-bs",
"4",
],
)
cls.base_url += "/v1"
@@ -149,6 +155,8 @@ class TestMinicpmvServer(TestOpenAIVisionServer):
"--trust-remote-code",
"--mem-fraction-static",
"0.35",
"--cuda-graph-max-bs",
"4",
],
)
cls.base_url += "/v1"
@@ -164,7 +172,11 @@ class TestInternVL2_5Server(TestOpenAIVisionServer):
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=["--trust-remote-code"],
other_args=[
"--trust-remote-code",
"--cuda-graph-max-bs",
"4",
],
)
cls.base_url += "/v1"
@@ -183,6 +195,8 @@ class TestMinicpmoServer(TestOpenAIVisionServer):
"--trust-remote-code",
"--mem-fraction-static",
"0.65",
"--cuda-graph-max-bs",
"4",
],
)
cls.base_url += "/v1"
@@ -207,10 +221,13 @@ class TestMimoVLServer(TestOpenAIVisionServer):
"--trust-remote-code",
"--mem-fraction-static",
"0.6",
"--cuda-graph-max-bs",
"4",
],
)
cls.base_url += "/v1"
if __name__ == "__main__":
del TestOpenAIVisionServer
unittest.main()

26
test/srt/test_vision_openai_server_b.py
View File

@@ -23,6 +23,8 @@ class TestPixtralServer(TestOpenAIVisionServer):
"--trust-remote-code",
"--mem-fraction-static",
"0.70",
"--cuda-graph-max-bs",
"4",
],
)
cls.base_url += "/v1"
@@ -45,6 +47,8 @@ class TestMistral3_1Server(TestOpenAIVisionServer):
"--trust-remote-code",
"--mem-fraction-static",
"0.75",
"--cuda-graph-max-bs",
"4",
],
)
cls.base_url += "/v1"
@@ -67,7 +71,8 @@ class TestDeepseekVL2Server(TestOpenAIVisionServer):
"--trust-remote-code",
"--context-length",
"4096",
"--disable-cuda-graph",
"--cuda-graph-max-bs",
"4",
],
)
cls.base_url += "/v1"
@@ -90,6 +95,8 @@ class TestJanusProServer(TestOpenAIVisionServer):
"--trust-remote-code",
"--mem-fraction-static",
"0.35",
"--cuda-graph-max-bs",
"4",
],
)
cls.base_url += "/v1"
@@ -120,6 +127,10 @@ class TestJanusProServer(TestOpenAIVisionServer):
# "0.8",
# "--tp-size=8",
# "--context-length=8192",
# "--mm-attention-backend",
# "fa3",
# "--cuda-graph-max-bs",
# "4",
# ],
# )
# cls.base_url += "/v1"
@@ -143,6 +154,8 @@ class TestGemma3itServer(TestOpenAIVisionServer):
"--mem-fraction-static",
"0.70",
"--enable-multimodal",
"--cuda-graph-max-bs",
"4",
],
)
cls.base_url += "/v1"
@@ -154,7 +167,7 @@ class TestGemma3itServer(TestOpenAIVisionServer):
class TestGemma3nServer(TestOpenAIVisionServer):
@classmethod
def setUpClass(cls):
cls.model = "google/gemma-3n-E2B-it"
cls.model = "google/gemma-3n-E4B-it"
cls.base_url = DEFAULT_URL_FOR_TEST
cls.api_key = "sk-123456"
cls.process = popen_launch_server(
@@ -166,7 +179,7 @@ class TestGemma3nServer(TestOpenAIVisionServer):
"--mem-fraction-static",
"0.70",
"--cuda-graph-max-bs",
"1",
"4",
],
)
cls.base_url += "/v1"
@@ -193,6 +206,8 @@ class TestKimiVLServer(TestOpenAIVisionServer):
"4096",
"--dtype",
"bfloat16",
"--cuda-graph-max-bs",
"4",
],
)
cls.base_url += "/v1"
@@ -233,6 +248,8 @@ class TestPhi4MMServer(TestOpenAIVisionServer):
"--lora-paths",
f"vision={constants.HF_HUB_CACHE}/models--microsoft--Phi-4-multimodal-instruct/snapshots/{revision}/vision-lora",
f"speech={constants.HF_HUB_CACHE}/models--microsoft--Phi-4-multimodal-instruct/snapshots/{revision}/speech-lora",
"--cuda-graph-max-bs",
"4",
],
)
cls.base_url += "/v1"
@@ -277,10 +294,13 @@ class TestVILAServer(TestOpenAIVisionServer):
"--trust-remote-code",
"--context-length=65536",
f"--revision={cls.revision}",
"--cuda-graph-max-bs",
"4",
],
)
cls.base_url += "/v1"
if __name__ == "__main__":
del TestOpenAIVisionServer
unittest.main()

24
test/srt/test_vision_openai_server_common.py
View File

@@ -71,7 +71,7 @@ class TestOpenAIVisionServer(CustomTestCase):
},
{
"type": "text",
"text": "Describe this image in a very short sentence.",
"text": "Describe this image in a sentence.",
},
],
},
@@ -119,7 +119,7 @@ class TestOpenAIVisionServer(CustomTestCase):
},
{
"type": "text",
"text": "Describe this image in a very short sentence.",
"text": "Describe this image in a sentence.",
},
],
},
@@ -455,7 +455,7 @@ class TestOpenAIVisionServer(CustomTestCase):
content.append(
{
"type": "text",
"text": "Describe this image in a very short sentence.",
"text": "Describe this image in a sentence.",
}
)
@@ -528,14 +528,20 @@ class TestOpenAIVisionServer(CustomTestCase):
# a fragment of Trump's speech
audio_response = self.get_audio_response(
AUDIO_TRUMP_SPEECH_URL,
"I have an audio sample. Please repeat the person's words",
"Listen to this audio and write down the audio transcription in English.",
category="speech",
)
assert "thank you" in audio_response
assert "it's a privilege to be here" in audio_response
assert "leader" in audio_response
assert "science" in audio_response
assert "art" in audio_response
check_list = [
"thank you",
"it's a privilege to be here",
"leader",
"science",
"art",
]
for check_word in check_list:
assert (
check_word in audio_response
), f"audio_response: ｜{audio_response}｜ should contain ｜{check_word}｜"
def _test_audio_ambient_completion(self):
# bird song