Add bench_server_latency.py (#1452)

2024-09-18 00:56:06 -07:00
parent 5752f25eef
commit 5e62a6b706
5 changed files with 210 additions and 15 deletions
--- a/python/sglang/bench_serving.py
+++ b/python/sglang/bench_serving.py
@@ -2,7 +2,7 @@
 # Adapted from https://github.com/vllm-project/vllm/blob/6366efc67b0aedd2c1721c14385370e50b297fb3/benchmarks/benchmark_serving.py

 """
-Benchmark online serving.
+Benchmark online serving with dynamic requests.

 Usage:
 python3 -m sglang.bench_serving --backend sglang --num-prompt 10