bench: Add MMMU benchmark for vLM (#3562)

2025-02-23 00:10:59 +08:00
parent 9087694006
commit 45205d88a0
9 changed files with 1026 additions and 7 deletions
--- a/benchmark/mmmu/prompt_format.yaml
+++ b/benchmark/mmmu/prompt_format.yaml
@@ -0,0 +1,15 @@
+task_instructions:
+- ""
+multi_choice_example_format:
+- "{}
+
+{}
+
+Answer with the option's letter from the given choices directly."
+
+short_ans_example_format:
+- "{}
+
+Answer the question using a single word or phrase."
+temperature:
+- 0