EngineX-Hygon/sglang
5
0
Fork 0
Code Issues Pull Requests Actions 7 Projects Releases Wiki Activity

Improve torch compile for fused moe (#2327)

Browse Source
This commit is contained in:
Lianmin Zheng
2024-12-03 01:58:25 -08:00
committed by GitHub
parent 83b340e371
commit 07ec07ad1f
6 changed files with 45 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
test/srt/test_srt_engine.py
View File

@@ -188,7 +188,7 @@ class TestSRTEngine(unittest.TestCase):
)
bench_args = BenchArgs(num_prompts=10)
result = throughput_test(server_args=server_args, bench_args=bench_args)
self.assertGreater(result["total_throughput"], 3500)
self.assertGreater(result["total_throughput"], 3000)
if __name__ == "__main__":

4
test/srt/test_torch_compile_moe.py
View File

@@ -14,7 +14,7 @@ from sglang.test.test_utils import (
)
class TestTorchCompile(unittest.TestCase):
class TestTorchCompileMoe(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.model = DEFAULT_SMALL_MOE_MODEL_NAME_FOR_TEST
@@ -23,7 +23,7 @@ class TestTorchCompile(unittest.TestCase):
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=["--enable-torch-compile", "--torch-compile-max-bs", "1"],
other_args=["--enable-torch-compile", "--torch-compile-max-bs", "8"],
)
@classmethod