Log if cuda graph is used & extend cuda graph capture to cuda-graph-max-bs (#6201)

Co-authored-by: SangBin Cho <rkooo567@gmail.com>
2025-05-12 00:17:33 -07:00
parent 7d3a3d4510
commit fba8eccd7e
27 changed files with 293 additions and 121 deletions
--- a/test/srt/test_fa3.py
+++ b/test/srt/test_fa3.py
@@ -111,7 +111,9 @@ class BaseFlashAttentionTest(CustomTestCase):

        if self.speculative_decode:
            server_info = requests.get(self.base_url + "/get_server_info")
-            avg_spec_accept_length = server_info.json()["avg_spec_accept_length"]
+            avg_spec_accept_length = server_info.json()["internal_states"][0][
+                "avg_spec_accept_length"
+            ]
            print(f"{avg_spec_accept_length=}")
            self.assertGreater(avg_spec_accept_length, self.spec_decode_threshold)