Organize Benchmark (#381)

benchmark/multi_document_qa/README.md

@@ -22,7 +22,7 @@ python3 bench_other.py --backend vllm --num-questions 64
 
 ### Benchmark guidance
 ```
-python3 bench_other.py --backend guidance --num-questions 32 --parallel 1
+python3 bench_other.py --backend guidance --num-questions 32 --parallel 1 --n-ctx 11000 --model-path path/to/code-llama/gguf
 ```
 
 

benchmark/multi_document_qa/bench_other.py

@@ -6,12 +6,7 @@ from functools import partial
 
 from tqdm import tqdm
 
-from sglang.test.test_utils import (
-    add_common_other_args_and_parse,
-    call_generate_lightllm,
-    call_generate_srt_raw,
-    call_generate_vllm,
-)
+from sglang.test.test_utils import add_common_other_args_and_parse, get_call_generate
 from sglang.utils import dump_state_text, read_jsonl
 
 USER_PREFIX = "[INST] "
@@ -60,40 +55,11 @@ def main(args):
     states = [None] * len(arguments)
 
     # Select backend
-    if args.backend == "lightllm":
-        url = f"{args.host}:{args.port}/generate"
-        generate = partial(call_generate_lightllm, url=url, temperature=0)
-    elif args.backend == "vllm":
-        url = f"{args.host}:{args.port}/generate"
-        generate = partial(call_generate_vllm, url=url, temperature=0)
-    elif args.backend == "srt-raw":
-        url = f"{args.host}:{args.port}/generate"
-        generate = partial(call_generate_srt_raw, url=url, temperature=0)
-    elif args.backend == "guidance":
-        from guidance import gen, models
-
-        model = models.LlamaCpp(
-            "/home/ubuntu/model_weights/CodeLlama-7b-instruct-hf.gguf",
-            n_gpu_layers=-1,
-            n_ctx=11000,
-        )
-
-        def generate(prompt, max_tokens, stop):
-            out = (
-                model
-                + prompt
-                + gen(name="answer", max_tokens=max_tokens, temperature=0, stop=stop)
-            )
-            return out["answer"]
-
-        # warmup
-        generate("Hello!", max_tokens=8, stop=None)
-    else:
-        raise ValueError(f"Invalid backend: {args.backend}")
+    call_generate = partial(get_call_generate(args), temperature=0)
 
     # Run requests
     def get_one_answer(i):
-        states[i] = multi_document_qa(generate=generate, **arguments[i])
+        states[i] = multi_document_qa(generate=call_generate, **arguments[i])
 
     tic = time.time()
     if args.parallel == 1:
@@ -101,7 +67,13 @@ def main(args):
             get_one_answer(i)
     else:
         with ThreadPoolExecutor(args.parallel) as executor:
-            executor.map(get_one_answer, list(range(len(labels))))
+            list(
+                tqdm(
+                    executor.map(get_one_answer, list(range(len(labels)))),
+                    total=len(labels),
+                )
+            )
+
     latency = time.time() - tic
 
     # Compute accuracy