EngineX-Hygon/sglang
5
0
Fork 0
Code Issues Pull Requests Actions 7 Projects Releases Wiki Activity

chore: improve mmmu benchmark (#7000)

Browse Source 
Signed-off-by: Xinyuan Tong <xinyuantong.cs@gmail.com>
Co-authored-by: Xinyuan Tong <xinyuantong.cs@gmail.com>
This commit is contained in:
Mick
2025-07-26 16:19:45 +08:00
committed by GitHub
parent e6312d271d
commit 4fa44d63c6
2 changed files with 23 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
benchmark/mmmu/eval_utils.py
View File

@@ -27,8 +27,7 @@ from tqdm import tqdm
class EvalArgs:
seed: int = 42
split: str = "validation"
# Default setting to make the benchmark available on A100 for most 7B models
image_pixels_limit: int = 4300000
image_pixels_limit: int = -1
result_filename: str = ""
prompt_format_file: str = "prompt_format.yaml"
dataset_path: str = "MMMU/MMMU"
@@ -190,7 +189,7 @@ def prepare_samples(eval_args: EvalArgs):
sample = construct_prompt(sample, eval_args.config)
image = sample["image"]
width, height = image.size
if width * height >= eval_args.image_pixels_limit:
if 0 < eval_args.image_pixels_limit <= width * height:
return None, True
# Use a unique identifier for the image path to avoid potential collisions if indices reset
image_path = f"{images_path}/image_{sample['id']}.png"
@@ -217,6 +216,8 @@ def prepare_samples(eval_args: EvalArgs):
elif sample:
samples.append(sample)
samples.sort(key=lambda x: x["final_input_prompt"])
print(
f"Skipping {skip_count} samples with large images, {round((float(skip_count) / len(dataset)) * 100, 2)}% of dataset"
)