Improve torch compile for fused moe (#2327)

2024-12-03 01:58:25 -08:00
parent 83b340e371
commit 07ec07ad1f
6 changed files with 45 additions and 24 deletions
--- a/test/srt/test_srt_engine.py
+++ b/test/srt/test_srt_engine.py
@@ -188,7 +188,7 @@ class TestSRTEngine(unittest.TestCase):
        )
        bench_args = BenchArgs(num_prompts=10)
        result = throughput_test(server_args=server_args, bench_args=bench_args)
-        self.assertGreater(result["total_throughput"], 3500)
+        self.assertGreater(result["total_throughput"], 3000)


 if __name__ == "__main__":