adapt to sglang v0.5.2rc1 on dcu
This commit is contained in:
maxiao
49
benchmark/multi_chain_reasoning/README.md
Normal file
49
benchmark/multi_chain_reasoning/README.md
Normal file
@@ -0,0 +1,49 @@
|
||||
## Download data
|
||||
```
|
||||
wget https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/test.jsonl
|
||||
```
|
||||
|
||||
## Run benchmark
|
||||
|
||||
### Benchmark sglang
|
||||
```
|
||||
python -m sglang.launch_server --model-path meta-llama/Llama-2-7b-chat-hf --port 30000 --schedule-conservativeness 1.3
|
||||
```
|
||||
|
||||
```
|
||||
python3 bench_sglang.py --num-questions 64
|
||||
python3 bench_sglang.py --num-questions 32 --parallel 1
|
||||
```
|
||||
|
||||
|
||||
### Benchmark vllm
|
||||
```
|
||||
python3 -m vllm.entrypoints.api_server --tokenizer-mode auto --model meta-llama/Llama-2-7b-chat-hf --disable-log-requests --port 21000
|
||||
```
|
||||
|
||||
```
|
||||
python3 bench_other.py --num-questions 64 --backend vllm
|
||||
```
|
||||
|
||||
|
||||
### Benchmark lightllm
|
||||
```
|
||||
# A10G
|
||||
python -m lightllm.server.api_server --tokenizer_mode auto --model_dir ~/model_weights/llama-2-7b-chat-hf --max_total_token_num 16000 --port 22000
|
||||
```
|
||||
|
||||
```
|
||||
python3 bench_other.py --num-questions 64 --backend lightllm
|
||||
```
|
||||
|
||||
|
||||
### Benchmark guidance
|
||||
```
|
||||
python3 bench_other.py --num-questions 8 --backend guidance --parallel 1 --n-ctx 4096 --model-path path/to/gguf
|
||||
```
|
||||
|
||||
### Benchmark lmql
|
||||
|
||||
```
|
||||
python3 bench_other.py --num-questions 64 --backend lmql --parallel 1
|
||||
```
|
||||
186
benchmark/multi_chain_reasoning/bench_other.py
Normal file
186
benchmark/multi_chain_reasoning/bench_other.py
Normal file
@@ -0,0 +1,186 @@
|
||||
import argparse
|
||||
import ast
|
||||
import asyncio
|
||||
import json
|
||||
import re
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
import numpy as np
|
||||
from tqdm import tqdm
|
||||
|
||||
from sglang.test.test_utils import add_common_other_args_and_parse, get_call_generate
|
||||
from sglang.utils import dump_state_text, read_jsonl
|
||||
|
||||
INVALID = -9999999
|
||||
|
||||
|
||||
def get_answer_value(answer_str):
|
||||
answer_str = answer_str.replace(",", "")
|
||||
numbers = re.findall(r"\d+", answer_str)
|
||||
if len(numbers) < 1:
|
||||
return INVALID
|
||||
try:
|
||||
return ast.literal_eval(numbers[-1])
|
||||
except SyntaxError:
|
||||
return INVALID
|
||||
|
||||
|
||||
prompt_lib = [
|
||||
"Let us think step by step.",
|
||||
"Approach this methodically. Let's dissect the problem into smaller, more manageable parts.",
|
||||
"It's important to proceed step by step, ensuring accuracy at each stage.",
|
||||
"Take a deep breath and break this down.",
|
||||
"A little bit of arithmetic and a logical approach will help us quickly arrive at the solution to this problem.",
|
||||
"I am extremely good at math.",
|
||||
]
|
||||
|
||||
|
||||
def multi_chain_gsm8k(question, num_chains, call_generate):
|
||||
s = "Question: " + question + "\n"
|
||||
# s += call_generate(s + "Answer: " + prompt_lib[0], max_tokens=256,
|
||||
# stop="Question", temperature=0)
|
||||
# return s
|
||||
|
||||
comps = []
|
||||
for i in range(num_chains):
|
||||
comps.append(
|
||||
call_generate(
|
||||
s + "Answer: " + prompt_lib[i % num_chains],
|
||||
max_tokens=256,
|
||||
temperature=0.3,
|
||||
stop="Question",
|
||||
)
|
||||
)
|
||||
|
||||
s += "Answer: To answer this question, here are some possible solutions. "
|
||||
s += "After considering all of them, I will do a majority vote.\n\n"
|
||||
for i in range(num_chains):
|
||||
s += f"Solution {i+1}: " + comps[i].strip() + "\n\n"
|
||||
s += "\nBy considering the above solutions and doing a majority vote, I think the final answer (a single integer number) is "
|
||||
s += call_generate(s, max_tokens=16, temperature=0, stop=None)
|
||||
return s
|
||||
|
||||
|
||||
async def multi_chain_gsm8k_async(question, num_chains, call_generate):
|
||||
s = "Question: " + question + "\n"
|
||||
# s += call_generate(s + "Answer: " + prompt_lib[0], max_tokens=256,
|
||||
# stop="Question", temperature=0)
|
||||
# return s
|
||||
|
||||
comps = []
|
||||
for i in range(num_chains):
|
||||
comps.append(
|
||||
await call_generate(
|
||||
s + "Answer: " + prompt_lib[i % num_chains],
|
||||
max_tokens=256,
|
||||
temperature=0.3,
|
||||
stop="Question",
|
||||
)
|
||||
)
|
||||
|
||||
s += "Answer: To answer this question, here are some possible solutions. "
|
||||
s += "After considering all of them, I will do a majority vote.\n\n"
|
||||
for i in range(num_chains):
|
||||
s += f"Solution {i+1}: " + comps[i].strip() + "\n\n"
|
||||
s += "\nBy considering the above solutions and doing a majority vote, I think the final answer (a single integer number) is "
|
||||
s += await call_generate(s, max_tokens=16, temperature=0, stop=None)
|
||||
return s
|
||||
|
||||
|
||||
def main(args):
|
||||
lines = read_jsonl(args.data_path)
|
||||
|
||||
# Construct prompts
|
||||
k = args.num_shot
|
||||
|
||||
questions = []
|
||||
labels = []
|
||||
for i in range(len(lines[: args.num_questions])):
|
||||
questions.append(lines[i]["question"])
|
||||
labels.append(get_answer_value(lines[i]["answer"]))
|
||||
assert all(l != INVALID for l in labels)
|
||||
|
||||
states = [None] * len(labels)
|
||||
|
||||
# Select backend
|
||||
call_generate = get_call_generate(args)
|
||||
|
||||
# Run requests
|
||||
if args.backend != "lmql":
|
||||
# Use thread pool
|
||||
def get_one_answer(i):
|
||||
answer = multi_chain_gsm8k(questions[i], args.num_chains, call_generate)
|
||||
states[i] = answer
|
||||
|
||||
tic = time.perf_counter()
|
||||
if args.parallel == 1:
|
||||
for i in tqdm(range(len(questions))):
|
||||
get_one_answer(i)
|
||||
else:
|
||||
with ThreadPoolExecutor(args.parallel) as executor:
|
||||
list(
|
||||
tqdm(
|
||||
executor.map(get_one_answer, list(range(len(questions)))),
|
||||
total=len(questions),
|
||||
)
|
||||
)
|
||||
|
||||
else:
|
||||
# Use asyncio
|
||||
async def get_one_answer_asyncio(i):
|
||||
answer = await multi_chain_gsm8k_async(
|
||||
questions[i], args.num_chains, call_generate
|
||||
)
|
||||
states[i] = answer
|
||||
|
||||
tic = time.perf_counter()
|
||||
loop = asyncio.get_event_loop()
|
||||
batches = [
|
||||
list(range(i, min(i + args.parallel, len(questions))))
|
||||
for i in range(0, len(questions), args.parallel)
|
||||
]
|
||||
for bt in tqdm(batches):
|
||||
tasks = [get_one_answer_asyncio(k) for k in bt]
|
||||
loop.run_until_complete(asyncio.gather(*tasks))
|
||||
|
||||
latency = time.perf_counter() - tic
|
||||
|
||||
preds = []
|
||||
for i in range(len(states)):
|
||||
preds.append(get_answer_value(states[i]))
|
||||
|
||||
# Compute accuracy
|
||||
acc = np.mean(np.array(preds) == np.array(labels))
|
||||
invalid = np.mean(np.array(preds) == INVALID)
|
||||
print(f"Latency: {latency:.3f}")
|
||||
print(f"Invalid: {invalid:.3f}")
|
||||
print(f"Accuracy: {acc:.3f}")
|
||||
|
||||
# Write results
|
||||
dump_state_text(f"tmp_output_{args.backend}.txt", states)
|
||||
|
||||
with open(args.result_file, "a") as fout:
|
||||
value = {
|
||||
"task": "multi_chain_gsm8k",
|
||||
"backend": args.backend,
|
||||
"num_gpus": 1,
|
||||
"latency": round(latency, 3),
|
||||
"accuracy": round(acc, 3),
|
||||
"num_requests": args.num_questions,
|
||||
"other": {
|
||||
"num_questions": args.num_questions,
|
||||
"parallel": args.parallel,
|
||||
},
|
||||
}
|
||||
fout.write(json.dumps(value) + "\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--num-shot", type=int, default=0)
|
||||
parser.add_argument("--num-chains", type=int, default=5)
|
||||
parser.add_argument("--data-path", type=str, default="test.jsonl")
|
||||
parser.add_argument("--num-questions", type=int, default=50)
|
||||
args = add_common_other_args_and_parse(parser)
|
||||
main(args)
|
||||
140
benchmark/multi_chain_reasoning/bench_sglang.py
Normal file
140
benchmark/multi_chain_reasoning/bench_sglang.py
Normal file
@@ -0,0 +1,140 @@
|
||||
import argparse
|
||||
import ast
|
||||
import json
|
||||
import re
|
||||
import time
|
||||
|
||||
import numpy as np
|
||||
|
||||
from sglang.test.test_utils import (
|
||||
add_common_sglang_args_and_parse,
|
||||
select_sglang_backend,
|
||||
)
|
||||
from sglang.utils import dump_state_text, read_jsonl
|
||||
|
||||
INVALID = -9999999
|
||||
|
||||
|
||||
def get_answer_value(answer_str):
|
||||
answer_str = answer_str.replace(",", "")
|
||||
numbers = re.findall(r"\d+", answer_str)
|
||||
if len(numbers) < 1:
|
||||
return INVALID
|
||||
try:
|
||||
return ast.literal_eval(numbers[-1])
|
||||
except SyntaxError:
|
||||
return INVALID
|
||||
|
||||
|
||||
prompt_lib = [
|
||||
"Let us think step by step.",
|
||||
"Approach this methodically. Let's dissect the problem into smaller, more manageable parts.",
|
||||
"It's important to proceed step by step, ensuring accuracy at each stage.",
|
||||
"Take a deep breath and break this down.",
|
||||
"A little bit of arithmetic and a logical approach will help us quickly arrive at the solution to this problem.",
|
||||
"I am extremely good at math.",
|
||||
]
|
||||
|
||||
|
||||
def main(args):
|
||||
lines = read_jsonl(args.data_path)
|
||||
|
||||
# Construct prompts
|
||||
# k = args.num_shot
|
||||
# few_shot_examples = get_few_shot_examples(lines, k)
|
||||
|
||||
questions = []
|
||||
labels = []
|
||||
for i in range(len(lines[: args.num_questions])):
|
||||
questions.append(lines[i]["question"])
|
||||
labels.append(get_answer_value(lines[i]["answer"]))
|
||||
assert all(l != INVALID for l in labels)
|
||||
arguments = [{"question": q} for q in questions]
|
||||
|
||||
num_chains = args.num_chains
|
||||
|
||||
#####################################
|
||||
######### SGL Program Begin #########
|
||||
#####################################
|
||||
|
||||
import sglang as sgl
|
||||
|
||||
@sgl.function
|
||||
def multi_chain_gsm8k(s, question):
|
||||
s += "Question: " + question + "\n"
|
||||
# s += "Answer: " + prompt_lib[0] + sgl.gen("answer", max_tokens=256, stop="Question",
|
||||
# temperature=0)
|
||||
# return
|
||||
|
||||
forks = s.fork(num_chains)
|
||||
for i in range(num_chains):
|
||||
forks[i] += (
|
||||
"Answer: "
|
||||
+ prompt_lib[i % num_chains]
|
||||
+ sgl.gen("chain", max_tokens=256, temperature=0.3, stop="Question")
|
||||
)
|
||||
forks.join()
|
||||
|
||||
s += "Answer: To answer this question, here are some possible solutions. "
|
||||
s += "After considering all of them, I will do a majority vote.\n\n"
|
||||
for i in range(num_chains):
|
||||
s += f"Solution {i+1}: " + forks[i]["chain"].strip() + "\n\n"
|
||||
s += "\nBy considering the above solutions and doing a majority vote, I think the final answer (a single integer number) is "
|
||||
s += sgl.gen("answer", max_tokens=16)
|
||||
|
||||
#####################################
|
||||
########## SGL Program End ##########
|
||||
#####################################
|
||||
|
||||
# Select backend
|
||||
backend = select_sglang_backend(args)
|
||||
|
||||
# Run requests
|
||||
tic = time.perf_counter()
|
||||
states = multi_chain_gsm8k.run_batch(
|
||||
arguments,
|
||||
temperature=0,
|
||||
backend=backend,
|
||||
num_threads=args.parallel,
|
||||
progress_bar=True,
|
||||
)
|
||||
latency = time.perf_counter() - tic
|
||||
|
||||
preds = []
|
||||
for i in range(len(states)):
|
||||
preds.append(get_answer_value(states[i]["answer"]))
|
||||
|
||||
# Compute accuracy
|
||||
acc = np.mean(np.array(preds) == np.array(labels))
|
||||
invalid = np.mean(np.array(preds) == INVALID)
|
||||
print(f"Latency: {latency:.3f}")
|
||||
print(f"Invalid: {invalid:.3f}")
|
||||
print(f"Accuracy: {acc:.3f}")
|
||||
|
||||
# Write results
|
||||
dump_state_text(f"tmp_output_{args.backend}.txt", states)
|
||||
|
||||
with open(args.result_file, "a") as fout:
|
||||
value = {
|
||||
"task": "multi_chain_gsm8k",
|
||||
"backend": args.backend,
|
||||
"num_gpus": 1,
|
||||
"latency": round(latency, 3),
|
||||
"accuracy": round(acc, 3),
|
||||
"num_requests": args.num_questions,
|
||||
"other": {
|
||||
"num_questions": args.num_questions,
|
||||
"parallel": args.parallel,
|
||||
},
|
||||
}
|
||||
fout.write(json.dumps(value) + "\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--num-shot", type=int, default=0)
|
||||
parser.add_argument("--num-chains", type=int, default=5)
|
||||
parser.add_argument("--data-path", type=str, default="test.jsonl")
|
||||
parser.add_argument("--num-questions", type=int, default=50)
|
||||
args = add_common_sglang_args_and_parse(parser)
|
||||
main(args)
|
||||
Reference in New Issue
Block a user