Sampler cudagraph (#1253)

2024-08-28 18:58:52 -07:00
parent 8153168c96
commit 381dd57bd6
29 changed files with 342 additions and 116 deletions
--- a/python/sglang/test/runners.py
+++ b/python/sglang/test/runners.py
@@ -180,7 +180,7 @@ class SRTRunner:
            tp_size=tp_size,
            dtype=get_dtype_str(torch_dtype),
            port=port,
-            mem_fraction_static=0.7,
+            mem_fraction_static=0.69,
            trust_remote_code=False,
            is_embedding=not self.is_generation,
        )