From 17000d2b3ad178f494fdc9309f64ab3d02c04b40 Mon Sep 17 00:00:00 2001 From: AinL Date: Fri, 28 Mar 2025 00:41:33 +0900 Subject: [PATCH] Remove Unintended Capture Batch Sizes in AMD HIP Graph Runner (#4638) --- python/sglang/srt/model_executor/cuda_graph_runner.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/sglang/srt/model_executor/cuda_graph_runner.py b/python/sglang/srt/model_executor/cuda_graph_runner.py index 95a4dd6af..449113c70 100644 --- a/python/sglang/srt/model_executor/cuda_graph_runner.py +++ b/python/sglang/srt/model_executor/cuda_graph_runner.py @@ -124,8 +124,8 @@ def get_batch_sizes_to_capture(model_runner: ModelRunner): # capture less. capture_bs = list(range(1, 9)) + list(range(9, 33, 2)) + [64, 96, 128, 160] - if _is_hip: - capture_bs += [i * 8 for i in range(21, 33)] + if _is_hip: + capture_bs += [i * 8 for i in range(21, 33)] if max(capture_bs) > model_runner.req_to_token_pool.size: # In some case (e.g., with a small GPU or --max-running-requests), the #max-running-requests