EngineX-Hygon/sglang
5
0
Fork 0
Code Issues Pull Requests Actions 7 Projects Releases Wiki Activity

fix bench_serving mishandling of internal states (#11376)

Browse Source 
Signed-off-by: Shahar Mor <smor@nvidia.com>
This commit is contained in:
shaharmor98
2025-10-09 14:24:50 +03:00
committed by GitHub
parent 8f2cd177af
commit fa7e2c3049
1 changed files with 9 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
python/sglang/bench_serving.py
View File

@@ -1837,6 +1837,10 @@ async def benchmark(
server_info_json = server_info.json()
if "decode" in server_info_json:
server_info_json = server_info_json["decode"][0]
if (
"internal_states" in server_info_json
and server_info_json["internal_states"]
):
accept_length = server_info_json["internal_states"][0].get(
"avg_spec_accept_length", None
)
@@ -1844,6 +1848,8 @@ async def benchmark(
accept_length = None
else:
accept_length = None
else:
accept_length = None
# Compute metrics and print results
benchmark_duration = time.perf_counter() - benchmark_start_time