diff --git a/python/sglang/srt/server_args.py b/python/sglang/srt/server_args.py index a05d67b7d..e09c911ec 100644 --- a/python/sglang/srt/server_args.py +++ b/python/sglang/srt/server_args.py @@ -1201,6 +1201,9 @@ class ServerArgs: ) if self.max_running_requests is None: self.max_running_requests = 48 + logger.warning( + "Max running requests is reset to 48 for speculative decoding." + ) if self.speculative_algorithm == "EAGLE" and self.enable_beta_spec: self.disable_overlap_schedule = False diff --git a/test/srt/run_suite.py b/test/srt/run_suite.py index 51356c803..e6b1419d7 100644 --- a/test/srt/run_suite.py +++ b/test/srt/run_suite.py @@ -181,6 +181,7 @@ suites = { ], "per-commit-8-gpu-h200-deepseek-v32": [ TestFile("test_deepseek_v32_basic.py", 275), + TestFile("test_deepseek_v32_mtp.py", 275), ], "vllm_dependency_test": [ TestFile("quant/test_awq.py", 163), diff --git a/test/srt/test_deepseek_v32_basic.py b/test/srt/test_deepseek_v32_basic.py index eac6c27bf..de56d9585 100644 --- a/test/srt/test_deepseek_v32_basic.py +++ b/test/srt/test_deepseek_v32_basic.py @@ -16,7 +16,7 @@ from sglang.test.test_utils import ( DEEPSEEK_V32_MODEL_PATH = "deepseek-ai/DeepSeek-V3.2-Exp" -class TestDeepseekV3Basic(CustomTestCase): +class TestDeepseekV32Basic(CustomTestCase): @classmethod def setUpClass(cls): cls.model = DEEPSEEK_V32_MODEL_PATH @@ -57,7 +57,7 @@ class TestDeepseekV3Basic(CustomTestCase): if is_in_ci(): write_github_step_summary( - f"### test_gsm8k (deepseek-v3)\n" f'{metrics["accuracy"]=:.3f}\n' + f"### test_gsm8k (deepseek-v32)\n" f'{metrics["accuracy"]=:.3f}\n' ) self.assertGreater(metrics["accuracy"], 0.935) @@ -69,7 +69,7 @@ class TestDeepseekV3Basic(CustomTestCase): if is_in_ci(): write_github_step_summary( - f"### test_bs_1_speed (deepseek-v3)\n" f"{speed=:.2f} token/s\n" + f"### test_bs_1_speed (deepseek-v32)\n" f"{speed=:.2f} token/s\n" ) self.assertGreater(speed, 50) diff --git a/test/srt/test_deepseek_v32_mtp.py b/test/srt/test_deepseek_v32_mtp.py new file mode 100644 index 000000000..1681b56ce --- /dev/null +++ b/test/srt/test_deepseek_v32_mtp.py @@ -0,0 +1,105 @@ +import unittest +from types import SimpleNamespace + +import requests + +from sglang.srt.utils import kill_process_tree +from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.send_one import BenchArgs, send_one_prompt +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + is_in_ci, + popen_launch_server, + write_github_step_summary, +) + +FULL_DEEPSEEK_V3_MODEL_PATH = "deepseek-ai/DeepSeek-V3.2-Exp" + + +class TestDeepseekV32MTP(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = FULL_DEEPSEEK_V3_MODEL_PATH + cls.base_url = DEFAULT_URL_FOR_TEST + other_args = [ + "--trust-remote-code", + "--tp", + "8", + "--dp", + "8", + "--enable-dp-attention", + "--speculative-algorithm", + "EAGLE", + "--speculative-num-steps", + "3", + "--speculative-eagle-topk", + "1", + "--speculative-num-draft-tokens", + "4", + "--mem-frac", + "0.7", + ] + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_a_gsm8k( + self, + ): # Append an "a" to make this test run first (alphabetically) to warm up the server + requests.get(self.base_url + "/flush_cache") + + args = SimpleNamespace( + num_shots=8, + data_path=None, + num_questions=1400, + parallel=1400, + max_new_tokens=512, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval_few_shot_gsm8k(args) + print(f"{metrics=}") + + server_info = requests.get(self.base_url + "/get_server_info") + avg_spec_accept_length = server_info.json()["internal_states"][0][ + "avg_spec_accept_length" + ] + print(f"{avg_spec_accept_length=}") + + if is_in_ci(): + write_github_step_summary( + f"### test_gsm8k (deepseek-v32 mtp)\n" + f'{metrics["accuracy"]=:.3f}\n' + f"{avg_spec_accept_length=:.2f}\n" + ) + self.assertGreater(metrics["accuracy"], 0.935) + self.assertGreater(avg_spec_accept_length, 2.9) + + def test_bs_1_speed(self): + args = BenchArgs(port=int(self.base_url.split(":")[-1]), max_new_tokens=2048) + acc_length, speed = send_one_prompt(args) + + print(f"{acc_length=:.2f} {speed=:.2f}") + + if is_in_ci(): + write_github_step_summary( + f"### test_bs_1_speed (deepseek-v32 mtp)\n" + f"{acc_length=:.2f}\n" + f"{speed=:.2f} token/s\n" + ) + + self.assertGreater(acc_length, 2.9) + self.assertGreater(speed, 75) + + +if __name__ == "__main__": + unittest.main()