Simplify the heuristics for setting --mem-fraction-static (#7054)

2025-06-10 19:01:39 -07:00
parent 0f218731e3
commit 6b12d6a8d5
3 changed files with 57 additions and 50 deletions
--- a/python/sglang/srt/model_executor/cuda_graph_runner.py
+++ b/python/sglang/srt/model_executor/cuda_graph_runner.py
@@ -153,7 +153,7 @@ def get_batch_sizes_to_capture(model_runner: ModelRunner):

        gpu_mem = get_device_memory_capacity()
        if gpu_mem is not None:
-            if gpu_mem > 90 * 1024:  # H200
+            if gpu_mem > 90 * 1024:  # H200, H20
                capture_bs += list(range(160, 257, 8))
            if gpu_mem > 160 * 1000:  # B200, MI300
                capture_bs += list(range(256, 513, 16))