EngineX-Hygon/sglang
5
0
Fork 0
Code Issues Pull Requests Actions 7 Projects Releases Wiki Activity

feat: update bench serving (#685)

Browse Source
This commit is contained in:
zhyncs
2024-07-21 16:46:58 +10:00
committed by GitHub
parent 5ad033a070
commit 0fdb3127a1
1 changed files with 9 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
python/sglang/bench_serving.py
View File

@@ -619,8 +619,11 @@ async def benchmark(
and metrics.output_throughput is not None and metrics.output_throughput is not None
): ):
result = { result = {
"backend": args.backend,
"dataset_name": args.dataset_name, "dataset_name": args.dataset_name,
"request_rate": request_rate, "request_rate": request_rate,
"total_input": metrics.total_input,
"total_output": metrics.total_output,
"median_ttft": metrics.median_ttft_ms, "median_ttft": metrics.median_ttft_ms,
"median_itl": metrics.mean_itl_ms, "median_itl": metrics.mean_itl_ms,
"output_token_throughput": metrics.output_throughput, "output_token_throughput": metrics.output_throughput,
@@ -674,8 +677,11 @@ async def benchmark(
def parse_request_rate_range(request_rate_range): def parse_request_rate_range(request_rate_range):
start, stop, step = map(int, request_rate_range.split(",")) if len(request_rate_range.split(",")) == 3:
return list(range(start, stop, step)) start, stop, step = map(int, request_rate_range.split(","))
return list(range(start, stop, step))
else:
return list(map(int, request_rate_range.split(",")))
def fire(args: argparse.Namespace): def fire(args: argparse.Namespace):
@@ -895,7 +901,7 @@ if __name__ == "__main__":
"--request-rate-range", "--request-rate-range",
type=str, type=str,
default="2,34,2", default="2,34,2",
help="Range of request rates in the format start,stop,step. Default is 2,34,2", help="Range of request rates in the format start,stop,step. Default is 2,34,2. It also supports a list of request rates, requiring the parameters to not equal three.",
) )
parser.add_argument("--output-file", type=str, help="Output JSONL file name.") parser.add_argument("--output-file", type=str, help="Output JSONL file name.")
parser.add_argument( parser.add_argument(