EngineX-Hygon/sglang
5
0
Fork 0
Code Issues Pull Requests Actions 7 Projects Releases Wiki Activity

[CI] test chunked prefill more (#5798)

Browse Source
This commit is contained in:
Lianmin Zheng
2025-04-28 10:57:17 -07:00
committed by GitHub
parent d73ddeb196
commit 849c83a0c0
15 changed files with 212 additions and 97 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
test/srt/test_bench_one_batch.py
View File

@@ -3,16 +3,28 @@ import unittest
from sglang.test.test_utils import (
DEFAULT_MODEL_NAME_FOR_TEST,
DEFAULT_MOE_MODEL_NAME_FOR_TEST,
DEFAULT_SMALL_MODEL_NAME_FOR_TEST,
CustomTestCase,
is_in_ci,
run_bench_offline_throughput,
run_bench_one_batch,
write_github_step_summary,
)
# We use `run_bench_offline_throughput`` instead of `run_bench_one_batch` for most cases
# because `run_bench_offline_throughput`` has overlap scheduler.
class TestBenchOneBatch(CustomTestCase):
def test_bs1_default(self):
def test_bs1_small(self):
output_throughput = run_bench_one_batch(
DEFAULT_SMALL_MODEL_NAME_FOR_TEST, ["--cuda-graph-max-bs", "2"]
)
self.assertGreater(output_throughput, 50)
def test_bs1_default(self):
output_throughput = run_bench_offline_throughput(
DEFAULT_MODEL_NAME_FOR_TEST, ["--cuda-graph-max-bs", "2"]
)
@@ -24,26 +36,26 @@ class TestBenchOneBatch(CustomTestCase):
self.assertGreater(output_throughput, 135)
def test_moe_tp2_bs1(self):
output_throughput = run_bench_one_batch(
output_throughput = run_bench_offline_throughput(
DEFAULT_MOE_MODEL_NAME_FOR_TEST, ["--tp", "2", "--cuda-graph-max-bs", "2"]
)
if is_in_ci():
write_github_step_summary(
f"### test_moe_tp2_bs1\n"
f"### test_moe_tp2_bs1 (Mixtral-8x7B)\n"
f"output_throughput: {output_throughput:.2f} token/s\n"
)
self.assertGreater(output_throughput, 125)
def test_torch_compile_tp2_bs1(self):
output_throughput = run_bench_one_batch(
output_throughput = run_bench_offline_throughput(
DEFAULT_MODEL_NAME_FOR_TEST,
["--tp", "2", "--enable-torch-compile", "--cuda-graph-max-bs", "2"],
)
if is_in_ci():
write_github_step_summary(
f"### test_torch_compile_tp2_bs1\n"
f"### test_torch_compile_tp2_bs1 (Mixtral-8x7B)\n"
f"output_throughput: {output_throughput:.2f} token/s\n"
)
self.assertGreater(output_throughput, 220)