Use cuda event wait and synchronization instead of busy waiting (#2089)

2024-11-19 00:21:46 -08:00
parent b110453802
commit b7a065eae3
6 changed files with 28 additions and 26 deletions
--- a/test/srt/test_large_max_new_tokens.py
+++ b/test/srt/test_large_max_new_tokens.py
@@ -38,7 +38,7 @@ class TestLargeMaxNewTokens(unittest.TestCase):
            api_key=cls.api_key,
            other_args=(
                "--max-total-token",
-                "1024",
+                "1536",
                "--context-len",
                "8192",
                "--decode-log-interval",