EngineX/xc-llm-ascend
3
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

[CI]Update accuracy report test (#1288)

Browse Source 
### What this PR does / why we need it?
Update accuracy report test
1. Add Record commit hashes and GitHub links for both vllm and
vllm-ascend in accuracy reports
2. Add accuracy result verification checks to ensure output correctness
3. Creat PR via forked repository workflow

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
dense-accuracy-test:
https://github.com/vllm-project/vllm-ascend/actions/runs/15745619485
create pr via forked repository workflow:
https://github.com/zhangxinyuehfad/vllm-ascend/actions/runs/15747013719/job/44385134080
accuracy report pr:
https://github.com/vllm-project/vllm-ascend/pull/1292

Currently, the accuracy report used is old and needs to be merged into
pr, retest, update new report, then close #1292 .


Signed-off-by: hfadzxy <starmoon_zhang@163.com>
This commit is contained in:
zhangxinyuehfad
2025-06-25 14:10:34 +08:00
committed by GitHub
parent 15df8be937
commit 0060886a37
3 changed files with 204 additions and 235 deletions
Download Patch File Download Diff File Expand all files Collapse all files

91
benchmarks/scripts/run_accuracy.py
View File

@@ -31,24 +31,44 @@ UNIMODAL_TASK = ["ceval-valid", "gsm8k"]
MULTIMODAL_NAME = ["Qwen/Qwen2.5-VL-7B-Instruct"]
MULTIMODAL_TASK = ["mmmu_val"]
batch_size_dict = {"ceval-valid": 1, "mmlu": 1, "gsm8k": "auto", "mmmu_val": 1}
BATCH_SIZE = {"ceval-valid": 1, "mmlu": 1, "gsm8k": "auto", "mmmu_val": 1}
MODEL_RUN_INFO = {
"Qwen/Qwen2.5-7B-Instruct":
("export MODEL_ARGS='pretrained={model}, max_model_len=4096,dtype=auto,tensor_parallel_size=2,gpu_memory_utilization=0.6'\n"
("export MODEL_ARGS='pretrained={model},max_model_len=4096,dtype=auto,tensor_parallel_size=2,gpu_memory_utilization=0.6'\n"
"lm_eval --model vllm --model_args $MODEL_ARGS --tasks {datasets} \ \n"
"--apply_chat_template --fewshot_as_multiturn --num_fewshot 5 --batch_size 1"
),
"Qwen/Qwen3-8B-Base":
("export MODEL_ARGS='pretrained={model}, max_model_len=4096,dtype=auto,tensor_parallel_size=2,gpu_memory_utilization=0.6'\n"
("export MODEL_ARGS='pretrained={model},max_model_len=4096,dtype=auto,tensor_parallel_size=2,gpu_memory_utilization=0.6'\n"
"lm_eval --model vllm --model_args $MODEL_ARGS --tasks {datasets} \ \n"
"--apply_chat_template --fewshot_as_multiturn --num_fewshot 5 --batch_size 1"
),
"Qwen/Qwen2.5-VL-7B-Instruct":
("export MODEL_ARGS='pretrained={model}, max_model_len=8192,dtype=auto,tensor_parallel_size=4,max_images=2'\n"
("export MODEL_ARGS='pretrained={model},max_model_len=8192,dtype=auto,tensor_parallel_size=4,max_images=2'\n"
"lm_eval --model vllm-vlm --model_args $MODEL_ARGS --tasks {datasets} \ \n"
"--apply_chat_template --fewshot_as_multiturn --batch_size 1"),
}
FILTER = {
"gsm8k": "exact_match,flexible-extract",
"ceval-valid": "acc,none",
"mmmu_val": "acc,none"
}
EXPECTED_VALUE = {
"Qwen/Qwen2.5-7B-Instruct": {
"ceval-valid": 0.80,
"gsm8k": 0.72
},
"Qwen/Qwen3-8B-Base": {
"ceval-valid": 0.82,
"gsm8k": 0.83
},
"Qwen/Qwen2.5-VL-7B-Instruct": {
"mmmu_val": 0.51
}
}
RTOL = 0.03
ACCURACY_FLAG = {}
def run_accuracy_unimodal(queue, model, dataset):
@@ -60,7 +80,7 @@ def run_accuracy_unimodal(queue, model, dataset):
tasks=dataset,
apply_chat_template=True,
fewshot_as_multiturn=True,
batch_size=batch_size_dict[dataset],
batch_size=BATCH_SIZE[dataset],
num_fewshot=5,
)
print(f"Success: {model} on {dataset}")
@@ -84,7 +104,7 @@ def run_accuracy_multimodal(queue, model, dataset):
tasks=dataset,
apply_chat_template=True,
fewshot_as_multiturn=True,
batch_size=batch_size_dict[dataset],
batch_size=BATCH_SIZE[dataset],
)
print(f"Success: {model} on {dataset}")
measured_value = results["results"]
@@ -102,25 +122,22 @@ def generate_md(model_name, tasks_list, args, datasets):
run_cmd = MODEL_RUN_INFO[model_name].format(model=model_name,
datasets=datasets)
model = model_name.split("/")[1]
preamble = f"""# 🎯 {model} Accuracy Test
<div>
<strong>vLLM version:</strong> vLLM: {args.vllm_version}, vLLM Ascend: {args.vllm_ascend_version} <br>
</div>
<div>
<strong>Software Environment:</strong> CANN: {args.cann_version}, PyTorch: {args.torch_version}, torch-npu: {args.torch_npu_version} <br>
</div>
<div>
<strong>Hardware Environment</strong>: Atlas A2 Series <br>
</div>
<div>
<strong>Datasets</strong>: {datasets} <br>
</div>
<div>
<strong>Command</strong>:
version_info = (
f"**vLLM Version**: vLLM: {args.vllm_version} "
f"([{args.vllm_commit}]({args.vllm_commit_url})), "
f"**vLLM Ascend**: {args.vllm_ascend_version} "
f"([{args.vllm_ascend_commit}]({args.vllm_ascend_commit_url}))")
```bash
{run_cmd}
```
preamble = f"""# 🎯 {model}
{version_info}
**vLLM Engine**: V{args.vllm_use_v1}
**Software Environment**: CANN: {args.cann_version}, PyTorch: {args.torch_version}, torch-npu: {args.torch_npu_version}
**Hardware Environment**: Atlas A2 Series
**Datasets**: {datasets}
**Command**:
```bash
{run_cmd}
```
</div>
<div>&nbsp;</div>
"""
@@ -153,11 +170,12 @@ def generate_md(model_name, tasks_list, args, datasets):
n_shot = "5"
else:
n_shot = "0"
flag = ACCURACY_FLAG.get(task_name, "")
row = (f"| {task_name:<37} "
f"| {flt:<6} "
f"| {n_shot:6} "
f"| {metric:<6} "
f"| ↑ {value:>5.4f} "
f"| {flag}{value:>5.4f} "
f"| ± {stderr:>5.4f} |")
if not task_name.startswith("-"):
rows.append(row)
@@ -187,6 +205,7 @@ def main(args):
if args.model in UNIMODAL_MODEL_NAME:
datasets = ",".join(UNIMODAL_TASK)
for dataset in UNIMODAL_TASK:
accuracy_expected = EXPECTED_VALUE[args.model][dataset]
p = multiprocessing.Process(target=run_accuracy_unimodal,
args=(result_queue, args.model,
dataset))
@@ -194,10 +213,16 @@ def main(args):
p.join()
result = result_queue.get()
print(result)
if accuracy_expected - RTOL < result[dataset][
FILTER[dataset]] < accuracy_expected + RTOL:
ACCURACY_FLAG[dataset] = "✅"
else:
ACCURACY_FLAG[dataset] = "❌"
accuracy[args.model].append(result)
if args.model in MULTIMODAL_NAME:
datasets = ",".join(MULTIMODAL_TASK)
for dataset in MULTIMODAL_TASK:
accuracy_expected = EXPECTED_VALUE[args.model][dataset]
p = multiprocessing.Process(target=run_accuracy_multimodal,
args=(result_queue, args.model,
dataset))
@@ -205,12 +230,18 @@ def main(args):
p.join()
result = result_queue.get()
print(result)
if accuracy_expected - RTOL < result[dataset][
FILTER[dataset]] < accuracy_expected + RTOL:
ACCURACY_FLAG[dataset] = "✅"
else:
ACCURACY_FLAG[dataset] = "❌"
accuracy[args.model].append(result)
print(accuracy)
safe_md(args, accuracy, datasets)
if __name__ == "__main__":
multiprocessing.set_start_method('spawn', force=True)
parser = argparse.ArgumentParser()
parser.add_argument("--output", type=str, required=True)
parser.add_argument("--model", type=str, required=True)
@@ -219,8 +250,12 @@ if __name__ == "__main__":
parser.add_argument("--torch_npu_version", type=str, required=False)
parser.add_argument("--vllm_version", type=str, required=False)
parser.add_argument("--cann_version", type=str, required=False)
parser.add_argument("--vllm_commit", type=lambda s: s[:7], required=False)
parser.add_argument("--vllm_commit_url", type=str, required=False)
parser.add_argument("--vllm_ascend_commit",
type=lambda s: s[:7],
required=False)
parser.add_argument("--vllm_ascend_commit_url", type=str, required=False)
parser.add_argument("--vllm_use_v1", type=str, required=False)
args = parser.parse_args()
# TODO(yikun):
# 1. add a exit 1 if accuracy is not as expected
# 2. Add ✅, ❌ to markdown if accuracy is not as expected
main(args)