Enable overlap by default (#2067)

2024-11-19 22:07:58 -08:00
parent 699384cb01
commit 7d671e4ad2
17 changed files with 92 additions and 75 deletions
--- a/test/srt/test_bench_serving.py
+++ b/test/srt/test_bench_serving.py
@@ -97,8 +97,8 @@ class TestBenchServing(unittest.TestCase):

        if is_in_ci():
            self.assertLess(res["median_e2e_latency_ms"], 12000)
-            self.assertLess(res["median_ttft_ms"], 80)
-            self.assertLess(res["median_itl_ms"], 11)
+            self.assertLess(res["median_ttft_ms"], 86)
+            self.assertLess(res["median_itl_ms"], 10)

    def test_moe_offline_throughput_default(self):
        res = run_bench_serving(