EngineX-Hygon/sglang
5
0
Fork 0
Code Issues Pull Requests Actions 7 Projects Releases Wiki Activity

Improve benchmark scripts (#717)

Browse Source
This commit is contained in:
Ying Sheng
2024-07-24 14:44:14 -07:00
committed by GitHub
parent 08a3bd19cc
commit 30d8e130e7
3 changed files with 6 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
python/sglang/bench_serving.py
View File

@@ -5,6 +5,9 @@ Benchmark online serving.
Usage:
python3 -m sglang.bench_serving --backend sglang --num-prompt 10
python3 -m sglang.bench_serving --backend sglang --dataset-name random --num-prompts 3000 --random-input 1024 --random-output 1024 --random-range-ratio 0.5
python3 -m sglang.bench_serving --backend sglang --dataset-name random --request-rate-range 1,2,4,8,16,32 --random-input 4096 --random-output 1024 --random-range-ratio 0.125 --multi
"""
import argparse

6
python/sglang/srt/managers/controller/tp_worker.py
View File

@@ -116,13 +116,9 @@ class ModelTpServer:
f"[gpu_id={self.gpu_id}] "
f"max_total_num_tokens={self.max_total_num_tokens}, "
f"max_prefill_tokens={self.max_prefill_tokens}, "
f"max_running_requests={self.max_running_requests}, "
f"context_len={self.model_config.context_len}"
)
if self.tp_rank == 0:
logger.info(
f"[gpu_id={self.gpu_id}] "
f"server_args: {server_args.print_mode_args()}"
)
# Init cache
self.tree_cache = RadixCache(