EngineX-Hygon/sglang
5
0
Fork 0
Code Issues Pull Requests Actions 7 Projects Releases Wiki Activity

Add accuracy and latency tests of eagle into CI (#3027)

Browse Source
This commit is contained in:
Lianmin Zheng
2025-01-21 02:55:14 -08:00
committed by GitHub
parent ec1c21cdc4
commit a4331cd260
7 changed files with 186 additions and 123 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
test/srt/test_bench_serving.py
View File

@@ -1,6 +1,8 @@
import unittest
from sglang.test.test_utils import (
DEFAULT_EAGLE_DRAFT_MODEL_FOR_TEST,
DEFAULT_EAGLE_TARGET_MODEL_FOR_TEST,
DEFAULT_FP8_MODEL_NAME_FOR_TEST,
DEFAULT_MODEL_NAME_FOR_TEST,
DEFAULT_MOE_MODEL_NAME_FOR_TEST,
@@ -47,7 +49,7 @@ class TestBenchServing(unittest.TestCase):
)
# There is a regression with torch 2.5
# This number was 950 for torch 2.4
self.assertGreater(res["output_throughput"], 800)
self.assertGreater(res["output_throughput"], 850)
def test_offline_throughput_without_radix_cache(self):
res = run_bench_serving(
@@ -131,6 +133,36 @@ class TestBenchServing(unittest.TestCase):
self.assertLess(res["median_ttft_ms"], 86)
self.assertLess(res["median_itl_ms"], 10)
def test_online_latency_eagle(self):
res = run_bench_serving(
model=DEFAULT_EAGLE_TARGET_MODEL_FOR_TEST,
num_prompts=50,
request_rate=1,
disable_ignore_eos=True,
dataset_name="sharegpt",
other_server_args=[
"--speculative-algorithm",
"EAGLE",
"--speculative-draft-model-path",
DEFAULT_EAGLE_DRAFT_MODEL_FOR_TEST,
"--speculative-num-steps",
"5",
"--speculative-eagle-topk",
"8",
"--speculative-num-draft-tokens",
"64",
"--mem-fraction-static",
"0.7",
],
)
if is_in_ci():
write_github_step_summary(
f"### test_online_latency_eagle\n"
f'median_e2e_latency_ms : {res["median_e2e_latency_ms"]:.2f} ms\n'
)
self.assertLess(res["median_e2e_latency_ms"], 10000)
def test_moe_offline_throughput_default(self):
res = run_bench_serving(
model=DEFAULT_MOE_MODEL_NAME_FOR_TEST,