EngineX-Hygon/sglang
5
0
Fork 0
Code Issues Pull Requests Actions 7 Projects Releases Wiki Activity

Fix typo in cuda_graph_bs (#2813)

Browse Source
This commit is contained in:
Lianmin Zheng
2025-01-09 03:03:24 -08:00
committed by GitHub
parent 656aed58c6
commit 679c3bcacf
1 changed files with 0 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
python/sglang/srt/model_executor/cuda_graph_runner.py
View File

@@ -131,11 +131,6 @@ class CudaGraphRunner:
else:
self.capture_bs = [1, 2, 4] + [i * 8 for i in range(1, 21)]
if model_runner.server_args.disable_cuda_graph_padding:
self.capture_bs = list(range(1, 33)) + [64, 128]
else:
self.capture_bs = [1, 2, 4] + [i * 8 for i in range(1, 21)]
if max(self.capture_bs) > model_runner.req_to_token_pool.size:
# In some case (e.g., with a small GPU or --max-running-requests), the #max-running-requests
# is very samll. We add more values here to make sure we capture the maximum bs.