Organize Benchmark (#381)

benchmark/mtbench/bench_other.py

@@ -4,16 +4,11 @@ import os
 import time
 import uuid
 from concurrent.futures import ThreadPoolExecutor
-from functools import partial
 
 from fastchat.model import get_conversation_template
+from tqdm import tqdm
 
-from sglang.test.test_utils import (
-    add_common_other_args_and_parse,
-    call_generate_lightllm,
-    call_generate_srt,
-    call_generate_vllm,
-)
+from sglang.test.test_utils import add_common_other_args_and_parse, get_call_generate
 
 
 def load_questions(filename):
@@ -50,17 +45,7 @@ def main(args):
     conv_main = get_conversation_template(model_id)
 
     # Select backend
-    if args.backend == "lightllm":
-        url = f"{args.host}:{args.port}/generate"
-        call_generate = partial(call_generate_lightllm, url=url, stop=None)
-    elif args.backend == "vllm":
-        url = f"{args.host}:{args.port}/generate"
-        call_generate = partial(call_generate_vllm, url=url, stop=None)
-    elif args.backend == "srt":
-        url = f"{args.host}:{args.port}/generate"
-        call_generate = partial(call_generate_srt, url=url, stop=None)
-    else:
-        raise ValueError(f"Invalid backend: {args.backend}")
+    call_generate = get_call_generate(args)
 
     answers = [None] * len(questions)
 
@@ -83,11 +68,17 @@ def main(args):
     # Run requests
     tic = time.time()
     if args.parallel == 1:
-        for i in range(len(questions)):
+        for i in tqdm(range(len(questions))):
             get_answer(i)
     else:
         with ThreadPoolExecutor(args.parallel) as executor:
-            executor.map(get_answer, list(range(len(questions))))
+            list(
+                tqdm(
+                    executor.map(get_answer, list(range(len(questions)))),
+                    total=len(questions),
+                )
+            )
+
     latency = time.time() - tic
 
     print(f"#questions: {len(questions)}, Latency: {latency:.2f}")