From 679c3bcacfd19eb852e8dbf42ad6b756eec56df4 Mon Sep 17 00:00:00 2001 From: Lianmin Zheng Date: Thu, 9 Jan 2025 03:03:24 -0800 Subject: [PATCH] Fix typo in cuda_graph_bs (#2813) --- python/sglang/srt/model_executor/cuda_graph_runner.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/python/sglang/srt/model_executor/cuda_graph_runner.py b/python/sglang/srt/model_executor/cuda_graph_runner.py index deaea3312..e4580b5e2 100644 --- a/python/sglang/srt/model_executor/cuda_graph_runner.py +++ b/python/sglang/srt/model_executor/cuda_graph_runner.py @@ -131,11 +131,6 @@ class CudaGraphRunner: else: self.capture_bs = [1, 2, 4] + [i * 8 for i in range(1, 21)] - if model_runner.server_args.disable_cuda_graph_padding: - self.capture_bs = list(range(1, 33)) + [64, 128] - else: - self.capture_bs = [1, 2, 4] + [i * 8 for i in range(1, 21)] - if max(self.capture_bs) > model_runner.req_to_token_pool.size: # In some case (e.g., with a small GPU or --max-running-requests), the #max-running-requests # is very samll. We add more values here to make sure we capture the maximum bs.