EngineX-Hygon/sglang
5
0
Fork 0
Code Issues Pull Requests Actions 7 Projects Releases Wiki Activity

Improve profiler and integrate profiler in bench_one_batch_server (#6787)

Browse Source
This commit is contained in:
Lianmin Zheng
2025-05-31 15:53:55 -07:00
committed by GitHub
parent b520d02888
commit 2d72fc47cf
25 changed files with 481 additions and 223 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
test/srt/test_bench_serving.py
View File

@@ -1,4 +1,3 @@
import os
import unittest
from sglang.test.test_utils import (
@@ -8,8 +7,8 @@ from sglang.test.test_utils import (
DEFAULT_MODEL_NAME_FOR_TEST_FP8,
DEFAULT_MOE_MODEL_NAME_FOR_TEST,
DEFAULT_SMALL_VLM_MODEL_NAME_FOR_TEST,
DEFAULT_VLM_CHAT_TEMPLATE_FOR_TEST,
CustomTestCase,
is_in_amd_ci,
is_in_ci,
run_bench_serving,
write_github_step_summary,
@@ -31,7 +30,7 @@ class TestBenchServing(CustomTestCase):
f"### test_offline_throughput_default\n"
f'Output throughput: {res["output_throughput"]:.2f} token/s\n'
)
if os.getenv("SGLANG_AMD_CI") == "1":
if is_in_amd_ci():
self.assertGreater(res["output_throughput"], 3150)
else:
self.assertGreater(res["output_throughput"], 3800)
@@ -69,7 +68,7 @@ class TestBenchServing(CustomTestCase):
f"### test_offline_throughput_without_radix_cache\n"
f'Output throughput: {res["output_throughput"]:.2f} token/s\n'
)
if os.getenv("SGLANG_AMD_CI") == "1":
if is_in_amd_ci():
self.assertGreater(res["output_throughput"], 3050)
else:
self.assertGreater(res["output_throughput"], 3800)
@@ -107,7 +106,7 @@ class TestBenchServing(CustomTestCase):
f"### test_offline_throughput_with_triton_attention_backend\n"
f'Output throughput: {res["output_throughput"]:.2f} token/s\n'
)
if os.getenv("SGLANG_AMD_CI") == "1":
if is_in_amd_ci():
self.assertGreater(res["output_throughput"], 3500)
else:
self.assertGreater(res["output_throughput"], 3700)
@@ -125,7 +124,7 @@ class TestBenchServing(CustomTestCase):
f"### test_offline_throughput_default_fp8\n"
f'Output throughput: {res["output_throughput"]:.2f} token/s\n'
)
if os.getenv("SGLANG_AMD_CI") == "1":
if is_in_amd_ci():
self.assertGreater(res["output_throughput"], 3500)
else:
self.assertGreater(res["output_throughput"], 4300)
@@ -144,7 +143,7 @@ class TestBenchServing(CustomTestCase):
f'median_e2e_latency_ms: {res["median_e2e_latency_ms"]:.2f} ms\n'
)
self.assertLess(res["median_e2e_latency_ms"], 11000)
if os.getenv("SGLANG_AMD_CI") == "1":
if is_in_amd_ci():
self.assertLess(res["median_ttft_ms"], 115)
else:
self.assertLess(res["median_ttft_ms"], 86)
@@ -167,7 +166,7 @@ class TestBenchServing(CustomTestCase):
f"### test_vlm_offline_throughput\n"
f'Output throughput: {res["output_throughput"]:.2f} token/s\n'
)
if os.getenv("SGLANG_AMD_CI") == "1":
if is_in_amd_ci():
self.assertGreater(res["output_throughput"], 2000)
# TODO: not set yet, need AMD machine
else:
@@ -191,7 +190,7 @@ class TestBenchServing(CustomTestCase):
f'median_e2e_latency_ms: {res["median_e2e_latency_ms"]:.2f} ms\n'
)
self.assertLess(res["median_e2e_latency_ms"], 16500)
if os.getenv("SGLANG_AMD_CI") == "1":
if is_in_amd_ci():
self.assertLess(res["median_ttft_ms"], 150)
# TODO: not set yet, need AMD machine
else:
@@ -230,7 +229,7 @@ class TestBenchServing(CustomTestCase):
f'median_e2e_latency_ms: {res["median_e2e_latency_ms"]:.2f} ms\n'
f'accept_length: {res["accept_length"]:.2f} \n'
)
if os.getenv("SGLANG_AMD_CI") == "1":
if is_in_amd_ci():
self.assertLess(res["median_e2e_latency_ms"], 1800)
else:
self.assertLess(res["median_e2e_latency_ms"], 900)
@@ -249,7 +248,7 @@ class TestBenchServing(CustomTestCase):
f"### test_moe_offline_throughput_default\n"
f'Output throughput: {res["output_throughput"]:.2f} token/s\n'
)
if os.getenv("SGLANG_AMD_CI") == "1":
if is_in_amd_ci():
self.assertGreater(res["output_throughput"], 2100)
else:
self.assertGreater(res["output_throughput"], 2200)
@@ -267,7 +266,7 @@ class TestBenchServing(CustomTestCase):
f"### test_moe_offline_throughput_without_radix_cache\n"
f'Output throughput: {res["output_throughput"]:.2f} token/s\n'
)
if os.getenv("SGLANG_AMD_CI") == "1":
if is_in_amd_ci():
self.assertGreater(res["output_throughput"], 2100)
else:
self.assertGreater(res["output_throughput"], 2200)
@@ -289,7 +288,7 @@ class TestBenchServing(CustomTestCase):
f"### test_pp_offline_throughput_default_decode\n"
f'Output throughput: {res["output_throughput"]:.2f} token/s\n'
)
self.assertGreater(res["output_throughput"], 7500)
self.assertGreater(res["output_throughput"], 6700)
def test_pp_long_context_prefill(self):
res = run_bench_serving(