4531 lines
139 KiB
JSON
4531 lines
139 KiB
JSON
|
{
|
||
|
|
"chat_template": "{%- if tools %}\n {{- '<|im_start|>system\\n' }}\n {%- if messages[0].role == 'system' %}\n {{- messages[0].content + '\\n\\n' }}\n {%- endif %}\n {{- \"# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n {%- for tool in tools %}\n {{- \"\\n\" }}\n {{- tool | tojson }}\n {%- endfor %}\n {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n {%- if messages[0].role == 'system' %}\n {{- '<|im_start|>system\\n' + messages[0].content + '<|im_end|>\\n' }}\n {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n {%- if message.content is string %}\n {%- set content = message.content %}\n {%- else %}\n {%- set content = '' %}\n {%- endif %}\n {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) %}\n {{- '<|im_start|>' + message.role + '\\n' + content + '<|im_end|>' + '\\n' }}\n {%- elif message.role == \"assistant\" %}\n {{- '<|im_start|>' + message.role + '\\n' + content }}\n {%- if message.tool_calls %}\n {%- for tool_call in message.tool_calls %}\n {%- if (loop.first and content) or (not loop.first) %}\n {{- '\\n' }}\n {%- endif %}\n {%- if tool_call.function %}\n {%- set tool_call = tool_call.function %}\n {%- endif %}\n {{- '<tool_call>\\n{\"name\": \"' }}\n {{- tool_call.name }}\n {{- '\", \"arguments\": ' }}\n {%- if tool_call.arguments is string %}\n {{- tool_call.arguments }}\n {%- else %}\n {{- tool_call.arguments | tojson }}\n {%- endif %}\n {{- '}\\n</tool_call>' }}\n {%- endfor %}\n {%- endif %}\n {{- '<|im_end|>\\n' }}\n {%- elif message.role == \"tool\" %}\n {%- if loop.first or (messages[loop.index0 - 1].role != \"tool\") %}\n {{- '<|im_start|>user' }}\n {%- endif %}\n {{- '\\n<tool_response>\\n' }}\n {{- content }}\n {{- '\\n</tool_response>' }}\n {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<|im_start|>assistant\\n' }}\n{%- endif %}",
|
||
|
|
"chat_template_sha": "64f85b198065d0fba2a81f37e10ed68161ce2c19a754c7100e67e0ca2ee9c326",
|
||
|
|
"config": {
|
||
|
|
"batch_size": "32",
|
||
|
|
"batch_sizes": [],
|
||
|
|
"bootstrap_iters": 100000,
|
||
|
|
"device": "cuda:0",
|
||
|
|
"fewshot_seed": 1234,
|
||
|
|
"gen_kwargs": {},
|
||
|
|
"limit": null,
|
||
|
|
"model": "vllm",
|
||
|
|
"model_args": "pretrained=Qwen/Qwen3-4B-Instruct-2507",
|
||
|
|
"numpy_seed": 1234,
|
||
|
|
"random_seed": 0,
|
||
|
|
"torch_seed": 1234,
|
||
|
|
"use_cache": "[redacted]"
|
||
|
|
},
|
||
|
|
"configs": {
|
||
|
|
"mmlu_abstract_algebra": {
|
||
|
|
"dataset_name": "abstract_algebra",
|
||
|
|
"dataset_path": "cais/mmlu",
|
||
|
|
"description": "The following are multiple choice questions (with answers) about abstract algebra.\n\n",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_config": {
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"sampler": "first_n",
|
||
|
|
"samples": null,
|
||
|
|
"split": "dev",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"metadata": {
|
||
|
|
"config_source": "[redacted]",
|
||
|
|
"dtype": "bfloat16",
|
||
|
|
"gpu_memory_utilization": 0.82,
|
||
|
|
"pretrained": "Qwen/Qwen3-4B-Instruct-2507",
|
||
|
|
"tensor_parallel_size": 1,
|
||
|
|
"trust_remote_code": true,
|
||
|
|
"version": 1.0
|
||
|
|
},
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true,
|
||
|
|
"metric": "acc"
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"task": "mmlu_abstract_algebra",
|
||
|
|
"task_alias": "abstract_algebra",
|
||
|
|
"test_split": "test",
|
||
|
|
"unsafe_code": false
|
||
|
|
},
|
||
|
|
"mmlu_anatomy": {
|
||
|
|
"dataset_name": "anatomy",
|
||
|
|
"dataset_path": "cais/mmlu",
|
||
|
|
"description": "The following are multiple choice questions (with answers) about anatomy.\n\n",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_config": {
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"sampler": "first_n",
|
||
|
|
"samples": null,
|
||
|
|
"split": "dev",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"metadata": {
|
||
|
|
"config_source": "[redacted]",
|
||
|
|
"dtype": "bfloat16",
|
||
|
|
"gpu_memory_utilization": 0.82,
|
||
|
|
"pretrained": "Qwen/Qwen3-4B-Instruct-2507",
|
||
|
|
"tensor_parallel_size": 1,
|
||
|
|
"trust_remote_code": true,
|
||
|
|
"version": 1.0
|
||
|
|
},
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true,
|
||
|
|
"metric": "acc"
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"task": "mmlu_anatomy",
|
||
|
|
"task_alias": "anatomy",
|
||
|
|
"test_split": "test",
|
||
|
|
"unsafe_code": false
|
||
|
|
},
|
||
|
|
"mmlu_astronomy": {
|
||
|
|
"dataset_name": "astronomy",
|
||
|
|
"dataset_path": "cais/mmlu",
|
||
|
|
"description": "The following are multiple choice questions (with answers) about astronomy.\n\n",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_config": {
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"sampler": "first_n",
|
||
|
|
"samples": null,
|
||
|
|
"split": "dev",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"metadata": {
|
||
|
|
"config_source": "[redacted]",
|
||
|
|
"dtype": "bfloat16",
|
||
|
|
"gpu_memory_utilization": 0.82,
|
||
|
|
"pretrained": "Qwen/Qwen3-4B-Instruct-2507",
|
||
|
|
"tensor_parallel_size": 1,
|
||
|
|
"trust_remote_code": true,
|
||
|
|
"version": 1.0
|
||
|
|
},
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true,
|
||
|
|
"metric": "acc"
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"task": "mmlu_astronomy",
|
||
|
|
"task_alias": "astronomy",
|
||
|
|
"test_split": "test",
|
||
|
|
"unsafe_code": false
|
||
|
|
},
|
||
|
|
"mmlu_business_ethics": {
|
||
|
|
"dataset_name": "business_ethics",
|
||
|
|
"dataset_path": "cais/mmlu",
|
||
|
|
"description": "The following are multiple choice questions (with answers) about business ethics.\n\n",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_config": {
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"sampler": "first_n",
|
||
|
|
"samples": null,
|
||
|
|
"split": "dev",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"metadata": {
|
||
|
|
"config_source": "[redacted]",
|
||
|
|
"dtype": "bfloat16",
|
||
|
|
"gpu_memory_utilization": 0.82,
|
||
|
|
"pretrained": "Qwen/Qwen3-4B-Instruct-2507",
|
||
|
|
"tensor_parallel_size": 1,
|
||
|
|
"trust_remote_code": true,
|
||
|
|
"version": 1.0
|
||
|
|
},
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true,
|
||
|
|
"metric": "acc"
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"task": "mmlu_business_ethics",
|
||
|
|
"task_alias": "business_ethics",
|
||
|
|
"test_split": "test",
|
||
|
|
"unsafe_code": false
|
||
|
|
},
|
||
|
|
"mmlu_clinical_knowledge": {
|
||
|
|
"dataset_name": "clinical_knowledge",
|
||
|
|
"dataset_path": "cais/mmlu",
|
||
|
|
"description": "The following are multiple choice questions (with answers) about clinical knowledge.\n\n",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_config": {
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"sampler": "first_n",
|
||
|
|
"samples": null,
|
||
|
|
"split": "dev",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"metadata": {
|
||
|
|
"config_source": "[redacted]",
|
||
|
|
"dtype": "bfloat16",
|
||
|
|
"gpu_memory_utilization": 0.82,
|
||
|
|
"pretrained": "Qwen/Qwen3-4B-Instruct-2507",
|
||
|
|
"tensor_parallel_size": 1,
|
||
|
|
"trust_remote_code": true,
|
||
|
|
"version": 1.0
|
||
|
|
},
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true,
|
||
|
|
"metric": "acc"
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"task": "mmlu_clinical_knowledge",
|
||
|
|
"task_alias": "clinical_knowledge",
|
||
|
|
"test_split": "test",
|
||
|
|
"unsafe_code": false
|
||
|
|
},
|
||
|
|
"mmlu_college_biology": {
|
||
|
|
"dataset_name": "college_biology",
|
||
|
|
"dataset_path": "cais/mmlu",
|
||
|
|
"description": "The following are multiple choice questions (with answers) about college biology.\n\n",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_config": {
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"sampler": "first_n",
|
||
|
|
"samples": null,
|
||
|
|
"split": "dev",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"metadata": {
|
||
|
|
"config_source": "[redacted]",
|
||
|
|
"dtype": "bfloat16",
|
||
|
|
"gpu_memory_utilization": 0.82,
|
||
|
|
"pretrained": "Qwen/Qwen3-4B-Instruct-2507",
|
||
|
|
"tensor_parallel_size": 1,
|
||
|
|
"trust_remote_code": true,
|
||
|
|
"version": 1.0
|
||
|
|
},
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true,
|
||
|
|
"metric": "acc"
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"task": "mmlu_college_biology",
|
||
|
|
"task_alias": "college_biology",
|
||
|
|
"test_split": "test",
|
||
|
|
"unsafe_code": false
|
||
|
|
},
|
||
|
|
"mmlu_college_chemistry": {
|
||
|
|
"dataset_name": "college_chemistry",
|
||
|
|
"dataset_path": "cais/mmlu",
|
||
|
|
"description": "The following are multiple choice questions (with answers) about college chemistry.\n\n",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_config": {
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"sampler": "first_n",
|
||
|
|
"samples": null,
|
||
|
|
"split": "dev",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"metadata": {
|
||
|
|
"config_source": "[redacted]",
|
||
|
|
"dtype": "bfloat16",
|
||
|
|
"gpu_memory_utilization": 0.82,
|
||
|
|
"pretrained": "Qwen/Qwen3-4B-Instruct-2507",
|
||
|
|
"tensor_parallel_size": 1,
|
||
|
|
"trust_remote_code": true,
|
||
|
|
"version": 1.0
|
||
|
|
},
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true,
|
||
|
|
"metric": "acc"
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"task": "mmlu_college_chemistry",
|
||
|
|
"task_alias": "college_chemistry",
|
||
|
|
"test_split": "test",
|
||
|
|
"unsafe_code": false
|
||
|
|
},
|
||
|
|
"mmlu_college_computer_science": {
|
||
|
|
"dataset_name": "college_computer_science",
|
||
|
|
"dataset_path": "cais/mmlu",
|
||
|
|
"description": "The following are multiple choice questions (with answers) about college computer science.\n\n",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_config": {
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"sampler": "first_n",
|
||
|
|
"samples": null,
|
||
|
|
"split": "dev",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"metadata": {
|
||
|
|
"config_source": "[redacted]",
|
||
|
|
"dtype": "bfloat16",
|
||
|
|
"gpu_memory_utilization": 0.82,
|
||
|
|
"pretrained": "Qwen/Qwen3-4B-Instruct-2507",
|
||
|
|
"tensor_parallel_size": 1,
|
||
|
|
"trust_remote_code": true,
|
||
|
|
"version": 1.0
|
||
|
|
},
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true,
|
||
|
|
"metric": "acc"
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"task": "mmlu_college_computer_science",
|
||
|
|
"task_alias": "college_computer_science",
|
||
|
|
"test_split": "test",
|
||
|
|
"unsafe_code": false
|
||
|
|
},
|
||
|
|
"mmlu_college_mathematics": {
|
||
|
|
"dataset_name": "college_mathematics",
|
||
|
|
"dataset_path": "cais/mmlu",
|
||
|
|
"description": "The following are multiple choice questions (with answers) about college mathematics.\n\n",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_config": {
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"sampler": "first_n",
|
||
|
|
"samples": null,
|
||
|
|
"split": "dev",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"metadata": {
|
||
|
|
"config_source": "[redacted]",
|
||
|
|
"dtype": "bfloat16",
|
||
|
|
"gpu_memory_utilization": 0.82,
|
||
|
|
"pretrained": "Qwen/Qwen3-4B-Instruct-2507",
|
||
|
|
"tensor_parallel_size": 1,
|
||
|
|
"trust_remote_code": true,
|
||
|
|
"version": 1.0
|
||
|
|
},
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true,
|
||
|
|
"metric": "acc"
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"task": "mmlu_college_mathematics",
|
||
|
|
"task_alias": "college_mathematics",
|
||
|
|
"test_split": "test",
|
||
|
|
"unsafe_code": false
|
||
|
|
},
|
||
|
|
"mmlu_college_medicine": {
|
||
|
|
"dataset_name": "college_medicine",
|
||
|
|
"dataset_path": "cais/mmlu",
|
||
|
|
"description": "The following are multiple choice questions (with answers) about college medicine.\n\n",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_config": {
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"sampler": "first_n",
|
||
|
|
"samples": null,
|
||
|
|
"split": "dev",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"metadata": {
|
||
|
|
"config_source": "[redacted]",
|
||
|
|
"dtype": "bfloat16",
|
||
|
|
"gpu_memory_utilization": 0.82,
|
||
|
|
"pretrained": "Qwen/Qwen3-4B-Instruct-2507",
|
||
|
|
"tensor_parallel_size": 1,
|
||
|
|
"trust_remote_code": true,
|
||
|
|
"version": 1.0
|
||
|
|
},
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true,
|
||
|
|
"metric": "acc"
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"task": "mmlu_college_medicine",
|
||
|
|
"task_alias": "college_medicine",
|
||
|
|
"test_split": "test",
|
||
|
|
"unsafe_code": false
|
||
|
|
},
|
||
|
|
"mmlu_college_physics": {
|
||
|
|
"dataset_name": "college_physics",
|
||
|
|
"dataset_path": "cais/mmlu",
|
||
|
|
"description": "The following are multiple choice questions (with answers) about college physics.\n\n",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_config": {
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"sampler": "first_n",
|
||
|
|
"samples": null,
|
||
|
|
"split": "dev",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"metadata": {
|
||
|
|
"config_source": "[redacted]",
|
||
|
|
"dtype": "bfloat16",
|
||
|
|
"gpu_memory_utilization": 0.82,
|
||
|
|
"pretrained": "Qwen/Qwen3-4B-Instruct-2507",
|
||
|
|
"tensor_parallel_size": 1,
|
||
|
|
"trust_remote_code": true,
|
||
|
|
"version": 1.0
|
||
|
|
},
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true,
|
||
|
|
"metric": "acc"
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"task": "mmlu_college_physics",
|
||
|
|
"task_alias": "college_physics",
|
||
|
|
"test_split": "test",
|
||
|
|
"unsafe_code": false
|
||
|
|
},
|
||
|
|
"mmlu_computer_security": {
|
||
|
|
"dataset_name": "computer_security",
|
||
|
|
"dataset_path": "cais/mmlu",
|
||
|
|
"description": "The following are multiple choice questions (with answers) about computer security.\n\n",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_config": {
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"sampler": "first_n",
|
||
|
|
"samples": null,
|
||
|
|
"split": "dev",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"metadata": {
|
||
|
|
"config_source": "[redacted]",
|
||
|
|
"dtype": "bfloat16",
|
||
|
|
"gpu_memory_utilization": 0.82,
|
||
|
|
"pretrained": "Qwen/Qwen3-4B-Instruct-2507",
|
||
|
|
"tensor_parallel_size": 1,
|
||
|
|
"trust_remote_code": true,
|
||
|
|
"version": 1.0
|
||
|
|
},
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true,
|
||
|
|
"metric": "acc"
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"task": "mmlu_computer_security",
|
||
|
|
"task_alias": "computer_security",
|
||
|
|
"test_split": "test",
|
||
|
|
"unsafe_code": false
|
||
|
|
},
|
||
|
|
"mmlu_conceptual_physics": {
|
||
|
|
"dataset_name": "conceptual_physics",
|
||
|
|
"dataset_path": "cais/mmlu",
|
||
|
|
"description": "The following are multiple choice questions (with answers) about conceptual physics.\n\n",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_config": {
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"sampler": "first_n",
|
||
|
|
"samples": null,
|
||
|
|
"split": "dev",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"metadata": {
|
||
|
|
"config_source": "[redacted]",
|
||
|
|
"dtype": "bfloat16",
|
||
|
|
"gpu_memory_utilization": 0.82,
|
||
|
|
"pretrained": "Qwen/Qwen3-4B-Instruct-2507",
|
||
|
|
"tensor_parallel_size": 1,
|
||
|
|
"trust_remote_code": true,
|
||
|
|
"version": 1.0
|
||
|
|
},
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true,
|
||
|
|
"metric": "acc"
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"task": "mmlu_conceptual_physics",
|
||
|
|
"task_alias": "conceptual_physics",
|
||
|
|
"test_split": "test",
|
||
|
|
"unsafe_code": false
|
||
|
|
},
|
||
|
|
"mmlu_econometrics": {
|
||
|
|
"dataset_name": "econometrics",
|
||
|
|
"dataset_path": "cais/mmlu",
|
||
|
|
"description": "The following are multiple choice questions (with answers) about econometrics.\n\n",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_config": {
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"sampler": "first_n",
|
||
|
|
"samples": null,
|
||
|
|
"split": "dev",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"metadata": {
|
||
|
|
"config_source": "[redacted]",
|
||
|
|
"dtype": "bfloat16",
|
||
|
|
"gpu_memory_utilization": 0.82,
|
||
|
|
"pretrained": "Qwen/Qwen3-4B-Instruct-2507",
|
||
|
|
"tensor_parallel_size": 1,
|
||
|
|
"trust_remote_code": true,
|
||
|
|
"version": 1.0
|
||
|
|
},
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true,
|
||
|
|
"metric": "acc"
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"task": "mmlu_econometrics",
|
||
|
|
"task_alias": "econometrics",
|
||
|
|
"test_split": "test",
|
||
|
|
"unsafe_code": false
|
||
|
|
},
|
||
|
|
"mmlu_electrical_engineering": {
|
||
|
|
"dataset_name": "electrical_engineering",
|
||
|
|
"dataset_path": "cais/mmlu",
|
||
|
|
"description": "The following are multiple choice questions (with answers) about electrical engineering.\n\n",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_config": {
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"sampler": "first_n",
|
||
|
|
"samples": null,
|
||
|
|
"split": "dev",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"metadata": {
|
||
|
|
"config_source": "[redacted]",
|
||
|
|
"dtype": "bfloat16",
|
||
|
|
"gpu_memory_utilization": 0.82,
|
||
|
|
"pretrained": "Qwen/Qwen3-4B-Instruct-2507",
|
||
|
|
"tensor_parallel_size": 1,
|
||
|
|
"trust_remote_code": true,
|
||
|
|
"version": 1.0
|
||
|
|
},
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true,
|
||
|
|
"metric": "acc"
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"task": "mmlu_electrical_engineering",
|
||
|
|
"task_alias": "electrical_engineering",
|
||
|
|
"test_split": "test",
|
||
|
|
"unsafe_code": false
|
||
|
|
},
|
||
|
|
"mmlu_elementary_mathematics": {
|
||
|
|
"dataset_name": "elementary_mathematics",
|
||
|
|
"dataset_path": "cais/mmlu",
|
||
|
|
"description": "The following are multiple choice questions (with answers) about elementary mathematics.\n\n",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_config": {
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"sampler": "first_n",
|
||
|
|
"samples": null,
|
||
|
|
"split": "dev",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"metadata": {
|
||
|
|
"config_source": "[redacted]",
|
||
|
|
"dtype": "bfloat16",
|
||
|
|
"gpu_memory_utilization": 0.82,
|
||
|
|
"pretrained": "Qwen/Qwen3-4B-Instruct-2507",
|
||
|
|
"tensor_parallel_size": 1,
|
||
|
|
"trust_remote_code": true,
|
||
|
|
"version": 1.0
|
||
|
|
},
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true,
|
||
|
|
"metric": "acc"
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"task": "mmlu_elementary_mathematics",
|
||
|
|
"task_alias": "elementary_mathematics",
|
||
|
|
"test_split": "test",
|
||
|
|
"unsafe_code": false
|
||
|
|
},
|
||
|
|
"mmlu_formal_logic": {
|
||
|
|
"dataset_name": "formal_logic",
|
||
|
|
"dataset_path": "cais/mmlu",
|
||
|
|
"description": "The following are multiple choice questions (with answers) about formal logic.\n\n",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_config": {
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"sampler": "first_n",
|
||
|
|
"samples": null,
|
||
|
|
"split": "dev",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"metadata": {
|
||
|
|
"config_source": "[redacted]",
|
||
|
|
"dtype": "bfloat16",
|
||
|
|
"gpu_memory_utilization": 0.82,
|
||
|
|
"pretrained": "Qwen/Qwen3-4B-Instruct-2507",
|
||
|
|
"tensor_parallel_size": 1,
|
||
|
|
"trust_remote_code": true,
|
||
|
|
"version": 1.0
|
||
|
|
},
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true,
|
||
|
|
"metric": "acc"
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"task": "mmlu_formal_logic",
|
||
|
|
"task_alias": "formal_logic",
|
||
|
|
"test_split": "test",
|
||
|
|
"unsafe_code": false
|
||
|
|
},
|
||
|
|
"mmlu_global_facts": {
|
||
|
|
"dataset_name": "global_facts",
|
||
|
|
"dataset_path": "cais/mmlu",
|
||
|
|
"description": "The following are multiple choice questions (with answers) about global facts.\n\n",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_config": {
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"sampler": "first_n",
|
||
|
|
"samples": null,
|
||
|
|
"split": "dev",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"metadata": {
|
||
|
|
"config_source": "[redacted]",
|
||
|
|
"dtype": "bfloat16",
|
||
|
|
"gpu_memory_utilization": 0.82,
|
||
|
|
"pretrained": "Qwen/Qwen3-4B-Instruct-2507",
|
||
|
|
"tensor_parallel_size": 1,
|
||
|
|
"trust_remote_code": true,
|
||
|
|
"version": 1.0
|
||
|
|
},
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true,
|
||
|
|
"metric": "acc"
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"task": "mmlu_global_facts",
|
||
|
|
"task_alias": "global_facts",
|
||
|
|
"test_split": "test",
|
||
|
|
"unsafe_code": false
|
||
|
|
},
|
||
|
|
"mmlu_high_school_biology": {
|
||
|
|
"dataset_name": "high_school_biology",
|
||
|
|
"dataset_path": "cais/mmlu",
|
||
|
|
"description": "The following are multiple choice questions (with answers) about high school biology.\n\n",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_config": {
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"sampler": "first_n",
|
||
|
|
"samples": null,
|
||
|
|
"split": "dev",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"metadata": {
|
||
|
|
"config_source": "[redacted]",
|
||
|
|
"dtype": "bfloat16",
|
||
|
|
"gpu_memory_utilization": 0.82,
|
||
|
|
"pretrained": "Qwen/Qwen3-4B-Instruct-2507",
|
||
|
|
"tensor_parallel_size": 1,
|
||
|
|
"trust_remote_code": true,
|
||
|
|
"version": 1.0
|
||
|
|
},
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true,
|
||
|
|
"metric": "acc"
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"task": "mmlu_high_school_biology",
|
||
|
|
"task_alias": "high_school_biology",
|
||
|
|
"test_split": "test",
|
||
|
|
"unsafe_code": false
|
||
|
|
},
|
||
|
|
"mmlu_high_school_chemistry": {
|
||
|
|
"dataset_name": "high_school_chemistry",
|
||
|
|
"dataset_path": "cais/mmlu",
|
||
|
|
"description": "The following are multiple choice questions (with answers) about high school chemistry.\n\n",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_config": {
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"sampler": "first_n",
|
||
|
|
"samples": null,
|
||
|
|
"split": "dev",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"metadata": {
|
||
|
|
"config_source": "[redacted]",
|
||
|
|
"dtype": "bfloat16",
|
||
|
|
"gpu_memory_utilization": 0.82,
|
||
|
|
"pretrained": "Qwen/Qwen3-4B-Instruct-2507",
|
||
|
|
"tensor_parallel_size": 1,
|
||
|
|
"trust_remote_code": true,
|
||
|
|
"version": 1.0
|
||
|
|
},
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true,
|
||
|
|
"metric": "acc"
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"task": "mmlu_high_school_chemistry",
|
||
|
|
"task_alias": "high_school_chemistry",
|
||
|
|
"test_split": "test",
|
||
|
|
"unsafe_code": false
|
||
|
|
},
|
||
|
|
"mmlu_high_school_computer_science": {
|
||
|
|
"dataset_name": "high_school_computer_science",
|
||
|
|
"dataset_path": "cais/mmlu",
|
||
|
|
"description": "The following are multiple choice questions (with answers) about high school computer science.\n\n",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_config": {
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"sampler": "first_n",
|
||
|
|
"samples": null,
|
||
|
|
"split": "dev",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"metadata": {
|
||
|
|
"config_source": "[redacted]",
|
||
|
|
"dtype": "bfloat16",
|
||
|
|
"gpu_memory_utilization": 0.82,
|
||
|
|
"pretrained": "Qwen/Qwen3-4B-Instruct-2507",
|
||
|
|
"tensor_parallel_size": 1,
|
||
|
|
"trust_remote_code": true,
|
||
|
|
"version": 1.0
|
||
|
|
},
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true,
|
||
|
|
"metric": "acc"
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"task": "mmlu_high_school_computer_science",
|
||
|
|
"task_alias": "high_school_computer_science",
|
||
|
|
"test_split": "test",
|
||
|
|
"unsafe_code": false
|
||
|
|
},
|
||
|
|
"mmlu_high_school_european_history": {
|
||
|
|
"dataset_name": "high_school_european_history",
|
||
|
|
"dataset_path": "cais/mmlu",
|
||
|
|
"description": "The following are multiple choice questions (with answers) about high school european history.\n\n",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_config": {
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"sampler": "first_n",
|
||
|
|
"samples": null,
|
||
|
|
"split": "dev",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"metadata": {
|
||
|
|
"config_source": "[redacted]",
|
||
|
|
"dtype": "bfloat16",
|
||
|
|
"gpu_memory_utilization": 0.82,
|
||
|
|
"pretrained": "Qwen/Qwen3-4B-Instruct-2507",
|
||
|
|
"tensor_parallel_size": 1,
|
||
|
|
"trust_remote_code": true,
|
||
|
|
"version": 1.0
|
||
|
|
},
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true,
|
||
|
|
"metric": "acc"
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"task": "mmlu_high_school_european_history",
|
||
|
|
"task_alias": "high_school_european_history",
|
||
|
|
"test_split": "test",
|
||
|
|
"unsafe_code": false
|
||
|
|
},
|
||
|
|
"mmlu_high_school_geography": {
|
||
|
|
"dataset_name": "high_school_geography",
|
||
|
|
"dataset_path": "cais/mmlu",
|
||
|
|
"description": "The following are multiple choice questions (with answers) about high school geography.\n\n",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_config": {
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"sampler": "first_n",
|
||
|
|
"samples": null,
|
||
|
|
"split": "dev",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"metadata": {
|
||
|
|
"config_source": "[redacted]",
|
||
|
|
"dtype": "bfloat16",
|
||
|
|
"gpu_memory_utilization": 0.82,
|
||
|
|
"pretrained": "Qwen/Qwen3-4B-Instruct-2507",
|
||
|
|
"tensor_parallel_size": 1,
|
||
|
|
"trust_remote_code": true,
|
||
|
|
"version": 1.0
|
||
|
|
},
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true,
|
||
|
|
"metric": "acc"
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"task": "mmlu_high_school_geography",
|
||
|
|
"task_alias": "high_school_geography",
|
||
|
|
"test_split": "test",
|
||
|
|
"unsafe_code": false
|
||
|
|
},
|
||
|
|
"mmlu_high_school_government_and_politics": {
|
||
|
|
"dataset_name": "high_school_government_and_politics",
|
||
|
|
"dataset_path": "cais/mmlu",
|
||
|
|
"description": "The following are multiple choice questions (with answers) about high school government and politics.\n\n",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_config": {
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"sampler": "first_n",
|
||
|
|
"samples": null,
|
||
|
|
"split": "dev",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"metadata": {
|
||
|
|
"config_source": "[redacted]",
|
||
|
|
"dtype": "bfloat16",
|
||
|
|
"gpu_memory_utilization": 0.82,
|
||
|
|
"pretrained": "Qwen/Qwen3-4B-Instruct-2507",
|
||
|
|
"tensor_parallel_size": 1,
|
||
|
|
"trust_remote_code": true,
|
||
|
|
"version": 1.0
|
||
|
|
},
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true,
|
||
|
|
"metric": "acc"
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"task": "mmlu_high_school_government_and_politics",
|
||
|
|
"task_alias": "high_school_government_and_politics",
|
||
|
|
"test_split": "test",
|
||
|
|
"unsafe_code": false
|
||
|
|
},
|
||
|
|
"mmlu_high_school_macroeconomics": {
|
||
|
|
"dataset_name": "high_school_macroeconomics",
|
||
|
|
"dataset_path": "cais/mmlu",
|
||
|
|
"description": "The following are multiple choice questions (with answers) about high school macroeconomics.\n\n",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_config": {
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"sampler": "first_n",
|
||
|
|
"samples": null,
|
||
|
|
"split": "dev",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"metadata": {
|
||
|
|
"config_source": "[redacted]",
|
||
|
|
"dtype": "bfloat16",
|
||
|
|
"gpu_memory_utilization": 0.82,
|
||
|
|
"pretrained": "Qwen/Qwen3-4B-Instruct-2507",
|
||
|
|
"tensor_parallel_size": 1,
|
||
|
|
"trust_remote_code": true,
|
||
|
|
"version": 1.0
|
||
|
|
},
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true,
|
||
|
|
"metric": "acc"
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"task": "mmlu_high_school_macroeconomics",
|
||
|
|
"task_alias": "high_school_macroeconomics",
|
||
|
|
"test_split": "test",
|
||
|
|
"unsafe_code": false
|
||
|
|
},
|
||
|
|
"mmlu_high_school_mathematics": {
|
||
|
|
"dataset_name": "high_school_mathematics",
|
||
|
|
"dataset_path": "cais/mmlu",
|
||
|
|
"description": "The following are multiple choice questions (with answers) about high school mathematics.\n\n",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_config": {
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"sampler": "first_n",
|
||
|
|
"samples": null,
|
||
|
|
"split": "dev",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"metadata": {
|
||
|
|
"config_source": "[redacted]",
|
||
|
|
"dtype": "bfloat16",
|
||
|
|
"gpu_memory_utilization": 0.82,
|
||
|
|
"pretrained": "Qwen/Qwen3-4B-Instruct-2507",
|
||
|
|
"tensor_parallel_size": 1,
|
||
|
|
"trust_remote_code": true,
|
||
|
|
"version": 1.0
|
||
|
|
},
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true,
|
||
|
|
"metric": "acc"
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"task": "mmlu_high_school_mathematics",
|
||
|
|
"task_alias": "high_school_mathematics",
|
||
|
|
"test_split": "test",
|
||
|
|
"unsafe_code": false
|
||
|
|
},
|
||
|
|
"mmlu_high_school_microeconomics": {
|
||
|
|
"dataset_name": "high_school_microeconomics",
|
||
|
|
"dataset_path": "cais/mmlu",
|
||
|
|
"description": "The following are multiple choice questions (with answers) about high school microeconomics.\n\n",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_config": {
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"sampler": "first_n",
|
||
|
|
"samples": null,
|
||
|
|
"split": "dev",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"metadata": {
|
||
|
|
"config_source": "[redacted]",
|
||
|
|
"dtype": "bfloat16",
|
||
|
|
"gpu_memory_utilization": 0.82,
|
||
|
|
"pretrained": "Qwen/Qwen3-4B-Instruct-2507",
|
||
|
|
"tensor_parallel_size": 1,
|
||
|
|
"trust_remote_code": true,
|
||
|
|
"version": 1.0
|
||
|
|
},
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true,
|
||
|
|
"metric": "acc"
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"task": "mmlu_high_school_microeconomics",
|
||
|
|
"task_alias": "high_school_microeconomics",
|
||
|
|
"test_split": "test",
|
||
|
|
"unsafe_code": false
|
||
|
|
},
|
||
|
|
"mmlu_high_school_physics": {
|
||
|
|
"dataset_name": "high_school_physics",
|
||
|
|
"dataset_path": "cais/mmlu",
|
||
|
|
"description": "The following are multiple choice questions (with answers) about high school physics.\n\n",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_config": {
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"sampler": "first_n",
|
||
|
|
"samples": null,
|
||
|
|
"split": "dev",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"metadata": {
|
||
|
|
"config_source": "[redacted]",
|
||
|
|
"dtype": "bfloat16",
|
||
|
|
"gpu_memory_utilization": 0.82,
|
||
|
|
"pretrained": "Qwen/Qwen3-4B-Instruct-2507",
|
||
|
|
"tensor_parallel_size": 1,
|
||
|
|
"trust_remote_code": true,
|
||
|
|
"version": 1.0
|
||
|
|
},
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true,
|
||
|
|
"metric": "acc"
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"task": "mmlu_high_school_physics",
|
||
|
|
"task_alias": "high_school_physics",
|
||
|
|
"test_split": "test",
|
||
|
|
"unsafe_code": false
|
||
|
|
},
|
||
|
|
"mmlu_high_school_psychology": {
|
||
|
|
"dataset_name": "high_school_psychology",
|
||
|
|
"dataset_path": "cais/mmlu",
|
||
|
|
"description": "The following are multiple choice questions (with answers) about high school psychology.\n\n",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_config": {
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"sampler": "first_n",
|
||
|
|
"samples": null,
|
||
|
|
"split": "dev",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"metadata": {
|
||
|
|
"config_source": "[redacted]",
|
||
|
|
"dtype": "bfloat16",
|
||
|
|
"gpu_memory_utilization": 0.82,
|
||
|
|
"pretrained": "Qwen/Qwen3-4B-Instruct-2507",
|
||
|
|
"tensor_parallel_size": 1,
|
||
|
|
"trust_remote_code": true,
|
||
|
|
"version": 1.0
|
||
|
|
},
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true,
|
||
|
|
"metric": "acc"
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"task": "mmlu_high_school_psychology",
|
||
|
|
"task_alias": "high_school_psychology",
|
||
|
|
"test_split": "test",
|
||
|
|
"unsafe_code": false
|
||
|
|
},
|
||
|
|
"mmlu_high_school_statistics": {
|
||
|
|
"dataset_name": "high_school_statistics",
|
||
|
|
"dataset_path": "cais/mmlu",
|
||
|
|
"description": "The following are multiple choice questions (with answers) about high school statistics.\n\n",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_config": {
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"sampler": "first_n",
|
||
|
|
"samples": null,
|
||
|
|
"split": "dev",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"metadata": {
|
||
|
|
"config_source": "[redacted]",
|
||
|
|
"dtype": "bfloat16",
|
||
|
|
"gpu_memory_utilization": 0.82,
|
||
|
|
"pretrained": "Qwen/Qwen3-4B-Instruct-2507",
|
||
|
|
"tensor_parallel_size": 1,
|
||
|
|
"trust_remote_code": true,
|
||
|
|
"version": 1.0
|
||
|
|
},
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true,
|
||
|
|
"metric": "acc"
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"task": "mmlu_high_school_statistics",
|
||
|
|
"task_alias": "high_school_statistics",
|
||
|
|
"test_split": "test",
|
||
|
|
"unsafe_code": false
|
||
|
|
},
|
||
|
|
"mmlu_high_school_us_history": {
|
||
|
|
"dataset_name": "high_school_us_history",
|
||
|
|
"dataset_path": "cais/mmlu",
|
||
|
|
"description": "The following are multiple choice questions (with answers) about high school us history.\n\n",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_config": {
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"sampler": "first_n",
|
||
|
|
"samples": null,
|
||
|
|
"split": "dev",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"metadata": {
|
||
|
|
"config_source": "[redacted]",
|
||
|
|
"dtype": "bfloat16",
|
||
|
|
"gpu_memory_utilization": 0.82,
|
||
|
|
"pretrained": "Qwen/Qwen3-4B-Instruct-2507",
|
||
|
|
"tensor_parallel_size": 1,
|
||
|
|
"trust_remote_code": true,
|
||
|
|
"version": 1.0
|
||
|
|
},
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true,
|
||
|
|
"metric": "acc"
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"task": "mmlu_high_school_us_history",
|
||
|
|
"task_alias": "high_school_us_history",
|
||
|
|
"test_split": "test",
|
||
|
|
"unsafe_code": false
|
||
|
|
},
|
||
|
|
"mmlu_high_school_world_history": {
|
||
|
|
"dataset_name": "high_school_world_history",
|
||
|
|
"dataset_path": "cais/mmlu",
|
||
|
|
"description": "The following are multiple choice questions (with answers) about high school world history.\n\n",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_config": {
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"sampler": "first_n",
|
||
|
|
"samples": null,
|
||
|
|
"split": "dev",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"metadata": {
|
||
|
|
"config_source": "[redacted]",
|
||
|
|
"dtype": "bfloat16",
|
||
|
|
"gpu_memory_utilization": 0.82,
|
||
|
|
"pretrained": "Qwen/Qwen3-4B-Instruct-2507",
|
||
|
|
"tensor_parallel_size": 1,
|
||
|
|
"trust_remote_code": true,
|
||
|
|
"version": 1.0
|
||
|
|
},
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true,
|
||
|
|
"metric": "acc"
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"task": "mmlu_high_school_world_history",
|
||
|
|
"task_alias": "high_school_world_history",
|
||
|
|
"test_split": "test",
|
||
|
|
"unsafe_code": false
|
||
|
|
},
|
||
|
|
"mmlu_human_aging": {
|
||
|
|
"dataset_name": "human_aging",
|
||
|
|
"dataset_path": "cais/mmlu",
|
||
|
|
"description": "The following are multiple choice questions (with answers) about human aging.\n\n",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_config": {
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"sampler": "first_n",
|
||
|
|
"samples": null,
|
||
|
|
"split": "dev",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"metadata": {
|
||
|
|
"config_source": "[redacted]",
|
||
|
|
"dtype": "bfloat16",
|
||
|
|
"gpu_memory_utilization": 0.82,
|
||
|
|
"pretrained": "Qwen/Qwen3-4B-Instruct-2507",
|
||
|
|
"tensor_parallel_size": 1,
|
||
|
|
"trust_remote_code": true,
|
||
|
|
"version": 1.0
|
||
|
|
},
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true,
|
||
|
|
"metric": "acc"
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"task": "mmlu_human_aging",
|
||
|
|
"task_alias": "human_aging",
|
||
|
|
"test_split": "test",
|
||
|
|
"unsafe_code": false
|
||
|
|
},
|
||
|
|
"mmlu_human_sexuality": {
|
||
|
|
"dataset_name": "human_sexuality",
|
||
|
|
"dataset_path": "cais/mmlu",
|
||
|
|
"description": "The following are multiple choice questions (with answers) about human sexuality.\n\n",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_config": {
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"sampler": "first_n",
|
||
|
|
"samples": null,
|
||
|
|
"split": "dev",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"metadata": {
|
||
|
|
"config_source": "[redacted]",
|
||
|
|
"dtype": "bfloat16",
|
||
|
|
"gpu_memory_utilization": 0.82,
|
||
|
|
"pretrained": "Qwen/Qwen3-4B-Instruct-2507",
|
||
|
|
"tensor_parallel_size": 1,
|
||
|
|
"trust_remote_code": true,
|
||
|
|
"version": 1.0
|
||
|
|
},
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true,
|
||
|
|
"metric": "acc"
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"task": "mmlu_human_sexuality",
|
||
|
|
"task_alias": "human_sexuality",
|
||
|
|
"test_split": "test",
|
||
|
|
"unsafe_code": false
|
||
|
|
},
|
||
|
|
"mmlu_international_law": {
|
||
|
|
"dataset_name": "international_law",
|
||
|
|
"dataset_path": "cais/mmlu",
|
||
|
|
"description": "The following are multiple choice questions (with answers) about international law.\n\n",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_config": {
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"sampler": "first_n",
|
||
|
|
"samples": null,
|
||
|
|
"split": "dev",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"metadata": {
|
||
|
|
"config_source": "[redacted]",
|
||
|
|
"dtype": "bfloat16",
|
||
|
|
"gpu_memory_utilization": 0.82,
|
||
|
|
"pretrained": "Qwen/Qwen3-4B-Instruct-2507",
|
||
|
|
"tensor_parallel_size": 1,
|
||
|
|
"trust_remote_code": true,
|
||
|
|
"version": 1.0
|
||
|
|
},
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true,
|
||
|
|
"metric": "acc"
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"task": "mmlu_international_law",
|
||
|
|
"task_alias": "international_law",
|
||
|
|
"test_split": "test",
|
||
|
|
"unsafe_code": false
|
||
|
|
},
|
||
|
|
"mmlu_jurisprudence": {
|
||
|
|
"dataset_name": "jurisprudence",
|
||
|
|
"dataset_path": "cais/mmlu",
|
||
|
|
"description": "The following are multiple choice questions (with answers) about jurisprudence.\n\n",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_config": {
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"sampler": "first_n",
|
||
|
|
"samples": null,
|
||
|
|
"split": "dev",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"metadata": {
|
||
|
|
"config_source": "[redacted]",
|
||
|
|
"dtype": "bfloat16",
|
||
|
|
"gpu_memory_utilization": 0.82,
|
||
|
|
"pretrained": "Qwen/Qwen3-4B-Instruct-2507",
|
||
|
|
"tensor_parallel_size": 1,
|
||
|
|
"trust_remote_code": true,
|
||
|
|
"version": 1.0
|
||
|
|
},
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true,
|
||
|
|
"metric": "acc"
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"task": "mmlu_jurisprudence",
|
||
|
|
"task_alias": "jurisprudence",
|
||
|
|
"test_split": "test",
|
||
|
|
"unsafe_code": false
|
||
|
|
},
|
||
|
|
"mmlu_logical_fallacies": {
|
||
|
|
"dataset_name": "logical_fallacies",
|
||
|
|
"dataset_path": "cais/mmlu",
|
||
|
|
"description": "The following are multiple choice questions (with answers) about logical fallacies.\n\n",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_config": {
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"sampler": "first_n",
|
||
|
|
"samples": null,
|
||
|
|
"split": "dev",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"metadata": {
|
||
|
|
"config_source": "[redacted]",
|
||
|
|
"dtype": "bfloat16",
|
||
|
|
"gpu_memory_utilization": 0.82,
|
||
|
|
"pretrained": "Qwen/Qwen3-4B-Instruct-2507",
|
||
|
|
"tensor_parallel_size": 1,
|
||
|
|
"trust_remote_code": true,
|
||
|
|
"version": 1.0
|
||
|
|
},
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true,
|
||
|
|
"metric": "acc"
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"task": "mmlu_logical_fallacies",
|
||
|
|
"task_alias": "logical_fallacies",
|
||
|
|
"test_split": "test",
|
||
|
|
"unsafe_code": false
|
||
|
|
},
|
||
|
|
"mmlu_machine_learning": {
|
||
|
|
"dataset_name": "machine_learning",
|
||
|
|
"dataset_path": "cais/mmlu",
|
||
|
|
"description": "The following are multiple choice questions (with answers) about machine learning.\n\n",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_config": {
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"sampler": "first_n",
|
||
|
|
"samples": null,
|
||
|
|
"split": "dev",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"metadata": {
|
||
|
|
"config_source": "[redacted]",
|
||
|
|
"dtype": "bfloat16",
|
||
|
|
"gpu_memory_utilization": 0.82,
|
||
|
|
"pretrained": "Qwen/Qwen3-4B-Instruct-2507",
|
||
|
|
"tensor_parallel_size": 1,
|
||
|
|
"trust_remote_code": true,
|
||
|
|
"version": 1.0
|
||
|
|
},
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true,
|
||
|
|
"metric": "acc"
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"task": "mmlu_machine_learning",
|
||
|
|
"task_alias": "machine_learning",
|
||
|
|
"test_split": "test",
|
||
|
|
"unsafe_code": false
|
||
|
|
},
|
||
|
|
"mmlu_management": {
|
||
|
|
"dataset_name": "management",
|
||
|
|
"dataset_path": "cais/mmlu",
|
||
|
|
"description": "The following are multiple choice questions (with answers) about management.\n\n",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_config": {
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"sampler": "first_n",
|
||
|
|
"samples": null,
|
||
|
|
"split": "dev",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"metadata": {
|
||
|
|
"config_source": "[redacted]",
|
||
|
|
"dtype": "bfloat16",
|
||
|
|
"gpu_memory_utilization": 0.82,
|
||
|
|
"pretrained": "Qwen/Qwen3-4B-Instruct-2507",
|
||
|
|
"tensor_parallel_size": 1,
|
||
|
|
"trust_remote_code": true,
|
||
|
|
"version": 1.0
|
||
|
|
},
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true,
|
||
|
|
"metric": "acc"
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"task": "mmlu_management",
|
||
|
|
"task_alias": "management",
|
||
|
|
"test_split": "test",
|
||
|
|
"unsafe_code": false
|
||
|
|
},
|
||
|
|
"mmlu_marketing": {
|
||
|
|
"dataset_name": "marketing",
|
||
|
|
"dataset_path": "cais/mmlu",
|
||
|
|
"description": "The following are multiple choice questions (with answers) about marketing.\n\n",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_config": {
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"sampler": "first_n",
|
||
|
|
"samples": null,
|
||
|
|
"split": "dev",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"metadata": {
|
||
|
|
"config_source": "[redacted]",
|
||
|
|
"dtype": "bfloat16",
|
||
|
|
"gpu_memory_utilization": 0.82,
|
||
|
|
"pretrained": "Qwen/Qwen3-4B-Instruct-2507",
|
||
|
|
"tensor_parallel_size": 1,
|
||
|
|
"trust_remote_code": true,
|
||
|
|
"version": 1.0
|
||
|
|
},
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true,
|
||
|
|
"metric": "acc"
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"task": "mmlu_marketing",
|
||
|
|
"task_alias": "marketing",
|
||
|
|
"test_split": "test",
|
||
|
|
"unsafe_code": false
|
||
|
|
},
|
||
|
|
"mmlu_medical_genetics": {
|
||
|
|
"dataset_name": "medical_genetics",
|
||
|
|
"dataset_path": "cais/mmlu",
|
||
|
|
"description": "The following are multiple choice questions (with answers) about medical genetics.\n\n",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_config": {
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"sampler": "first_n",
|
||
|
|
"samples": null,
|
||
|
|
"split": "dev",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"metadata": {
|
||
|
|
"config_source": "[redacted]",
|
||
|
|
"dtype": "bfloat16",
|
||
|
|
"gpu_memory_utilization": 0.82,
|
||
|
|
"pretrained": "Qwen/Qwen3-4B-Instruct-2507",
|
||
|
|
"tensor_parallel_size": 1,
|
||
|
|
"trust_remote_code": true,
|
||
|
|
"version": 1.0
|
||
|
|
},
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true,
|
||
|
|
"metric": "acc"
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"task": "mmlu_medical_genetics",
|
||
|
|
"task_alias": "medical_genetics",
|
||
|
|
"test_split": "test",
|
||
|
|
"unsafe_code": false
|
||
|
|
},
|
||
|
|
"mmlu_miscellaneous": {
|
||
|
|
"dataset_name": "miscellaneous",
|
||
|
|
"dataset_path": "cais/mmlu",
|
||
|
|
"description": "The following are multiple choice questions (with answers) about miscellaneous.\n\n",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_config": {
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"sampler": "first_n",
|
||
|
|
"samples": null,
|
||
|
|
"split": "dev",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"metadata": {
|
||
|
|
"config_source": "[redacted]",
|
||
|
|
"dtype": "bfloat16",
|
||
|
|
"gpu_memory_utilization": 0.82,
|
||
|
|
"pretrained": "Qwen/Qwen3-4B-Instruct-2507",
|
||
|
|
"tensor_parallel_size": 1,
|
||
|
|
"trust_remote_code": true,
|
||
|
|
"version": 1.0
|
||
|
|
},
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true,
|
||
|
|
"metric": "acc"
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"task": "mmlu_miscellaneous",
|
||
|
|
"task_alias": "miscellaneous",
|
||
|
|
"test_split": "test",
|
||
|
|
"unsafe_code": false
|
||
|
|
},
|
||
|
|
"mmlu_moral_disputes": {
|
||
|
|
"dataset_name": "moral_disputes",
|
||
|
|
"dataset_path": "cais/mmlu",
|
||
|
|
"description": "The following are multiple choice questions (with answers) about moral disputes.\n\n",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_config": {
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"sampler": "first_n",
|
||
|
|
"samples": null,
|
||
|
|
"split": "dev",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"metadata": {
|
||
|
|
"config_source": "[redacted]",
|
||
|
|
"dtype": "bfloat16",
|
||
|
|
"gpu_memory_utilization": 0.82,
|
||
|
|
"pretrained": "Qwen/Qwen3-4B-Instruct-2507",
|
||
|
|
"tensor_parallel_size": 1,
|
||
|
|
"trust_remote_code": true,
|
||
|
|
"version": 1.0
|
||
|
|
},
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true,
|
||
|
|
"metric": "acc"
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"task": "mmlu_moral_disputes",
|
||
|
|
"task_alias": "moral_disputes",
|
||
|
|
"test_split": "test",
|
||
|
|
"unsafe_code": false
|
||
|
|
},
|
||
|
|
"mmlu_moral_scenarios": {
|
||
|
|
"dataset_name": "moral_scenarios",
|
||
|
|
"dataset_path": "cais/mmlu",
|
||
|
|
"description": "The following are multiple choice questions (with answers) about moral scenarios.\n\n",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_config": {
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"sampler": "first_n",
|
||
|
|
"samples": null,
|
||
|
|
"split": "dev",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"metadata": {
|
||
|
|
"config_source": "[redacted]",
|
||
|
|
"dtype": "bfloat16",
|
||
|
|
"gpu_memory_utilization": 0.82,
|
||
|
|
"pretrained": "Qwen/Qwen3-4B-Instruct-2507",
|
||
|
|
"tensor_parallel_size": 1,
|
||
|
|
"trust_remote_code": true,
|
||
|
|
"version": 1.0
|
||
|
|
},
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true,
|
||
|
|
"metric": "acc"
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"task": "mmlu_moral_scenarios",
|
||
|
|
"task_alias": "moral_scenarios",
|
||
|
|
"test_split": "test",
|
||
|
|
"unsafe_code": false
|
||
|
|
},
|
||
|
|
"mmlu_nutrition": {
|
||
|
|
"dataset_name": "nutrition",
|
||
|
|
"dataset_path": "cais/mmlu",
|
||
|
|
"description": "The following are multiple choice questions (with answers) about nutrition.\n\n",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_config": {
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"sampler": "first_n",
|
||
|
|
"samples": null,
|
||
|
|
"split": "dev",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"metadata": {
|
||
|
|
"config_source": "[redacted]",
|
||
|
|
"dtype": "bfloat16",
|
||
|
|
"gpu_memory_utilization": 0.82,
|
||
|
|
"pretrained": "Qwen/Qwen3-4B-Instruct-2507",
|
||
|
|
"tensor_parallel_size": 1,
|
||
|
|
"trust_remote_code": true,
|
||
|
|
"version": 1.0
|
||
|
|
},
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true,
|
||
|
|
"metric": "acc"
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"task": "mmlu_nutrition",
|
||
|
|
"task_alias": "nutrition",
|
||
|
|
"test_split": "test",
|
||
|
|
"unsafe_code": false
|
||
|
|
},
|
||
|
|
"mmlu_philosophy": {
|
||
|
|
"dataset_name": "philosophy",
|
||
|
|
"dataset_path": "cais/mmlu",
|
||
|
|
"description": "The following are multiple choice questions (with answers) about philosophy.\n\n",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_config": {
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"sampler": "first_n",
|
||
|
|
"samples": null,
|
||
|
|
"split": "dev",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"metadata": {
|
||
|
|
"config_source": "[redacted]",
|
||
|
|
"dtype": "bfloat16",
|
||
|
|
"gpu_memory_utilization": 0.82,
|
||
|
|
"pretrained": "Qwen/Qwen3-4B-Instruct-2507",
|
||
|
|
"tensor_parallel_size": 1,
|
||
|
|
"trust_remote_code": true,
|
||
|
|
"version": 1.0
|
||
|
|
},
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true,
|
||
|
|
"metric": "acc"
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"task": "mmlu_philosophy",
|
||
|
|
"task_alias": "philosophy",
|
||
|
|
"test_split": "test",
|
||
|
|
"unsafe_code": false
|
||
|
|
},
|
||
|
|
"mmlu_prehistory": {
|
||
|
|
"dataset_name": "prehistory",
|
||
|
|
"dataset_path": "cais/mmlu",
|
||
|
|
"description": "The following are multiple choice questions (with answers) about prehistory.\n\n",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_config": {
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"sampler": "first_n",
|
||
|
|
"samples": null,
|
||
|
|
"split": "dev",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"metadata": {
|
||
|
|
"config_source": "[redacted]",
|
||
|
|
"dtype": "bfloat16",
|
||
|
|
"gpu_memory_utilization": 0.82,
|
||
|
|
"pretrained": "Qwen/Qwen3-4B-Instruct-2507",
|
||
|
|
"tensor_parallel_size": 1,
|
||
|
|
"trust_remote_code": true,
|
||
|
|
"version": 1.0
|
||
|
|
},
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true,
|
||
|
|
"metric": "acc"
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"task": "mmlu_prehistory",
|
||
|
|
"task_alias": "prehistory",
|
||
|
|
"test_split": "test",
|
||
|
|
"unsafe_code": false
|
||
|
|
},
|
||
|
|
"mmlu_professional_accounting": {
|
||
|
|
"dataset_name": "professional_accounting",
|
||
|
|
"dataset_path": "cais/mmlu",
|
||
|
|
"description": "The following are multiple choice questions (with answers) about professional accounting.\n\n",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_config": {
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"sampler": "first_n",
|
||
|
|
"samples": null,
|
||
|
|
"split": "dev",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"metadata": {
|
||
|
|
"config_source": "[redacted]",
|
||
|
|
"dtype": "bfloat16",
|
||
|
|
"gpu_memory_utilization": 0.82,
|
||
|
|
"pretrained": "Qwen/Qwen3-4B-Instruct-2507",
|
||
|
|
"tensor_parallel_size": 1,
|
||
|
|
"trust_remote_code": true,
|
||
|
|
"version": 1.0
|
||
|
|
},
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true,
|
||
|
|
"metric": "acc"
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"task": "mmlu_professional_accounting",
|
||
|
|
"task_alias": "professional_accounting",
|
||
|
|
"test_split": "test",
|
||
|
|
"unsafe_code": false
|
||
|
|
},
|
||
|
|
"mmlu_professional_law": {
|
||
|
|
"dataset_name": "professional_law",
|
||
|
|
"dataset_path": "cais/mmlu",
|
||
|
|
"description": "The following are multiple choice questions (with answers) about professional law.\n\n",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_config": {
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"sampler": "first_n",
|
||
|
|
"samples": null,
|
||
|
|
"split": "dev",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"metadata": {
|
||
|
|
"config_source": "[redacted]",
|
||
|
|
"dtype": "bfloat16",
|
||
|
|
"gpu_memory_utilization": 0.82,
|
||
|
|
"pretrained": "Qwen/Qwen3-4B-Instruct-2507",
|
||
|
|
"tensor_parallel_size": 1,
|
||
|
|
"trust_remote_code": true,
|
||
|
|
"version": 1.0
|
||
|
|
},
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true,
|
||
|
|
"metric": "acc"
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"task": "mmlu_professional_law",
|
||
|
|
"task_alias": "professional_law",
|
||
|
|
"test_split": "test",
|
||
|
|
"unsafe_code": false
|
||
|
|
},
|
||
|
|
"mmlu_professional_medicine": {
|
||
|
|
"dataset_name": "professional_medicine",
|
||
|
|
"dataset_path": "cais/mmlu",
|
||
|
|
"description": "The following are multiple choice questions (with answers) about professional medicine.\n\n",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_config": {
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"sampler": "first_n",
|
||
|
|
"samples": null,
|
||
|
|
"split": "dev",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"metadata": {
|
||
|
|
"config_source": "[redacted]",
|
||
|
|
"dtype": "bfloat16",
|
||
|
|
"gpu_memory_utilization": 0.82,
|
||
|
|
"pretrained": "Qwen/Qwen3-4B-Instruct-2507",
|
||
|
|
"tensor_parallel_size": 1,
|
||
|
|
"trust_remote_code": true,
|
||
|
|
"version": 1.0
|
||
|
|
},
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true,
|
||
|
|
"metric": "acc"
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"task": "mmlu_professional_medicine",
|
||
|
|
"task_alias": "professional_medicine",
|
||
|
|
"test_split": "test",
|
||
|
|
"unsafe_code": false
|
||
|
|
},
|
||
|
|
"mmlu_professional_psychology": {
|
||
|
|
"dataset_name": "professional_psychology",
|
||
|
|
"dataset_path": "cais/mmlu",
|
||
|
|
"description": "The following are multiple choice questions (with answers) about professional psychology.\n\n",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_config": {
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"sampler": "first_n",
|
||
|
|
"samples": null,
|
||
|
|
"split": "dev",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"metadata": {
|
||
|
|
"config_source": "[redacted]",
|
||
|
|
"dtype": "bfloat16",
|
||
|
|
"gpu_memory_utilization": 0.82,
|
||
|
|
"pretrained": "Qwen/Qwen3-4B-Instruct-2507",
|
||
|
|
"tensor_parallel_size": 1,
|
||
|
|
"trust_remote_code": true,
|
||
|
|
"version": 1.0
|
||
|
|
},
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true,
|
||
|
|
"metric": "acc"
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"task": "mmlu_professional_psychology",
|
||
|
|
"task_alias": "professional_psychology",
|
||
|
|
"test_split": "test",
|
||
|
|
"unsafe_code": false
|
||
|
|
},
|
||
|
|
"mmlu_public_relations": {
|
||
|
|
"dataset_name": "public_relations",
|
||
|
|
"dataset_path": "cais/mmlu",
|
||
|
|
"description": "The following are multiple choice questions (with answers) about public relations.\n\n",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_config": {
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"sampler": "first_n",
|
||
|
|
"samples": null,
|
||
|
|
"split": "dev",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"metadata": {
|
||
|
|
"config_source": "[redacted]",
|
||
|
|
"dtype": "bfloat16",
|
||
|
|
"gpu_memory_utilization": 0.82,
|
||
|
|
"pretrained": "Qwen/Qwen3-4B-Instruct-2507",
|
||
|
|
"tensor_parallel_size": 1,
|
||
|
|
"trust_remote_code": true,
|
||
|
|
"version": 1.0
|
||
|
|
},
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true,
|
||
|
|
"metric": "acc"
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"task": "mmlu_public_relations",
|
||
|
|
"task_alias": "public_relations",
|
||
|
|
"test_split": "test",
|
||
|
|
"unsafe_code": false
|
||
|
|
},
|
||
|
|
"mmlu_security_studies": {
|
||
|
|
"dataset_name": "security_studies",
|
||
|
|
"dataset_path": "cais/mmlu",
|
||
|
|
"description": "The following are multiple choice questions (with answers) about security studies.\n\n",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_config": {
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"sampler": "first_n",
|
||
|
|
"samples": null,
|
||
|
|
"split": "dev",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"metadata": {
|
||
|
|
"config_source": "[redacted]",
|
||
|
|
"dtype": "bfloat16",
|
||
|
|
"gpu_memory_utilization": 0.82,
|
||
|
|
"pretrained": "Qwen/Qwen3-4B-Instruct-2507",
|
||
|
|
"tensor_parallel_size": 1,
|
||
|
|
"trust_remote_code": true,
|
||
|
|
"version": 1.0
|
||
|
|
},
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true,
|
||
|
|
"metric": "acc"
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"task": "mmlu_security_studies",
|
||
|
|
"task_alias": "security_studies",
|
||
|
|
"test_split": "test",
|
||
|
|
"unsafe_code": false
|
||
|
|
},
|
||
|
|
"mmlu_sociology": {
|
||
|
|
"dataset_name": "sociology",
|
||
|
|
"dataset_path": "cais/mmlu",
|
||
|
|
"description": "The following are multiple choice questions (with answers) about sociology.\n\n",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_config": {
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"sampler": "first_n",
|
||
|
|
"samples": null,
|
||
|
|
"split": "dev",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"metadata": {
|
||
|
|
"config_source": "[redacted]",
|
||
|
|
"dtype": "bfloat16",
|
||
|
|
"gpu_memory_utilization": 0.82,
|
||
|
|
"pretrained": "Qwen/Qwen3-4B-Instruct-2507",
|
||
|
|
"tensor_parallel_size": 1,
|
||
|
|
"trust_remote_code": true,
|
||
|
|
"version": 1.0
|
||
|
|
},
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true,
|
||
|
|
"metric": "acc"
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"task": "mmlu_sociology",
|
||
|
|
"task_alias": "sociology",
|
||
|
|
"test_split": "test",
|
||
|
|
"unsafe_code": false
|
||
|
|
},
|
||
|
|
"mmlu_us_foreign_policy": {
|
||
|
|
"dataset_name": "us_foreign_policy",
|
||
|
|
"dataset_path": "cais/mmlu",
|
||
|
|
"description": "The following are multiple choice questions (with answers) about us foreign policy.\n\n",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_config": {
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"sampler": "first_n",
|
||
|
|
"samples": null,
|
||
|
|
"split": "dev",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"metadata": {
|
||
|
|
"config_source": "[redacted]",
|
||
|
|
"dtype": "bfloat16",
|
||
|
|
"gpu_memory_utilization": 0.82,
|
||
|
|
"pretrained": "Qwen/Qwen3-4B-Instruct-2507",
|
||
|
|
"tensor_parallel_size": 1,
|
||
|
|
"trust_remote_code": true,
|
||
|
|
"version": 1.0
|
||
|
|
},
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true,
|
||
|
|
"metric": "acc"
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"task": "mmlu_us_foreign_policy",
|
||
|
|
"task_alias": "us_foreign_policy",
|
||
|
|
"test_split": "test",
|
||
|
|
"unsafe_code": false
|
||
|
|
},
|
||
|
|
"mmlu_virology": {
|
||
|
|
"dataset_name": "virology",
|
||
|
|
"dataset_path": "cais/mmlu",
|
||
|
|
"description": "The following are multiple choice questions (with answers) about virology.\n\n",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_config": {
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"sampler": "first_n",
|
||
|
|
"samples": null,
|
||
|
|
"split": "dev",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"metadata": {
|
||
|
|
"config_source": "[redacted]",
|
||
|
|
"dtype": "bfloat16",
|
||
|
|
"gpu_memory_utilization": 0.82,
|
||
|
|
"pretrained": "Qwen/Qwen3-4B-Instruct-2507",
|
||
|
|
"tensor_parallel_size": 1,
|
||
|
|
"trust_remote_code": true,
|
||
|
|
"version": 1.0
|
||
|
|
},
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true,
|
||
|
|
"metric": "acc"
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"task": "mmlu_virology",
|
||
|
|
"task_alias": "virology",
|
||
|
|
"test_split": "test",
|
||
|
|
"unsafe_code": false
|
||
|
|
},
|
||
|
|
"mmlu_world_religions": {
|
||
|
|
"dataset_name": "world_religions",
|
||
|
|
"dataset_path": "cais/mmlu",
|
||
|
|
"description": "The following are multiple choice questions (with answers) about world religions.\n\n",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_config": {
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"sampler": "first_n",
|
||
|
|
"samples": null,
|
||
|
|
"split": "dev",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"metadata": {
|
||
|
|
"config_source": "[redacted]",
|
||
|
|
"dtype": "bfloat16",
|
||
|
|
"gpu_memory_utilization": 0.82,
|
||
|
|
"pretrained": "Qwen/Qwen3-4B-Instruct-2507",
|
||
|
|
"tensor_parallel_size": 1,
|
||
|
|
"trust_remote_code": true,
|
||
|
|
"version": 1.0
|
||
|
|
},
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true,
|
||
|
|
"metric": "acc"
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"task": "mmlu_world_religions",
|
||
|
|
"task_alias": "world_religions",
|
||
|
|
"test_split": "test",
|
||
|
|
"unsafe_code": false
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"date": 1781175138.5280895,
|
||
|
|
"eot_token_id": 151645,
|
||
|
|
"fewshot_as_multiturn": true,
|
||
|
|
"git_hash": null,
|
||
|
|
"group_subtasks": {
|
||
|
|
"mmlu": [
|
||
|
|
"mmlu_stem",
|
||
|
|
"mmlu_other",
|
||
|
|
"mmlu_social_sciences",
|
||
|
|
"mmlu_humanities"
|
||
|
|
],
|
||
|
|
"mmlu_humanities": [
|
||
|
|
"mmlu_formal_logic",
|
||
|
|
"mmlu_high_school_european_history",
|
||
|
|
"mmlu_high_school_us_history",
|
||
|
|
"mmlu_high_school_world_history",
|
||
|
|
"mmlu_international_law",
|
||
|
|
"mmlu_jurisprudence",
|
||
|
|
"mmlu_logical_fallacies",
|
||
|
|
"mmlu_moral_disputes",
|
||
|
|
"mmlu_moral_scenarios",
|
||
|
|
"mmlu_philosophy",
|
||
|
|
"mmlu_prehistory",
|
||
|
|
"mmlu_professional_law",
|
||
|
|
"mmlu_world_religions"
|
||
|
|
],
|
||
|
|
"mmlu_other": [
|
||
|
|
"mmlu_business_ethics",
|
||
|
|
"mmlu_clinical_knowledge",
|
||
|
|
"mmlu_college_medicine",
|
||
|
|
"mmlu_global_facts",
|
||
|
|
"mmlu_human_aging",
|
||
|
|
"mmlu_management",
|
||
|
|
"mmlu_marketing",
|
||
|
|
"mmlu_medical_genetics",
|
||
|
|
"mmlu_miscellaneous",
|
||
|
|
"mmlu_nutrition",
|
||
|
|
"mmlu_professional_accounting",
|
||
|
|
"mmlu_professional_medicine",
|
||
|
|
"mmlu_virology"
|
||
|
|
],
|
||
|
|
"mmlu_social_sciences": [
|
||
|
|
"mmlu_econometrics",
|
||
|
|
"mmlu_high_school_geography",
|
||
|
|
"mmlu_high_school_government_and_politics",
|
||
|
|
"mmlu_high_school_macroeconomics",
|
||
|
|
"mmlu_high_school_microeconomics",
|
||
|
|
"mmlu_high_school_psychology",
|
||
|
|
"mmlu_human_sexuality",
|
||
|
|
"mmlu_professional_psychology",
|
||
|
|
"mmlu_public_relations",
|
||
|
|
"mmlu_security_studies",
|
||
|
|
"mmlu_sociology",
|
||
|
|
"mmlu_us_foreign_policy"
|
||
|
|
],
|
||
|
|
"mmlu_stem": [
|
||
|
|
"mmlu_abstract_algebra",
|
||
|
|
"mmlu_anatomy",
|
||
|
|
"mmlu_astronomy",
|
||
|
|
"mmlu_college_biology",
|
||
|
|
"mmlu_college_chemistry",
|
||
|
|
"mmlu_college_computer_science",
|
||
|
|
"mmlu_college_mathematics",
|
||
|
|
"mmlu_college_physics",
|
||
|
|
"mmlu_computer_security",
|
||
|
|
"mmlu_conceptual_physics",
|
||
|
|
"mmlu_electrical_engineering",
|
||
|
|
"mmlu_elementary_mathematics",
|
||
|
|
"mmlu_high_school_biology",
|
||
|
|
"mmlu_high_school_chemistry",
|
||
|
|
"mmlu_high_school_computer_science",
|
||
|
|
"mmlu_high_school_mathematics",
|
||
|
|
"mmlu_high_school_physics",
|
||
|
|
"mmlu_high_school_statistics",
|
||
|
|
"mmlu_machine_learning"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
"groups": {
|
||
|
|
"mmlu": {
|
||
|
|
"acc,none": 0.6189289275032047,
|
||
|
|
"acc_stderr,none": 0.003746138843326914,
|
||
|
|
"alias": "mmlu",
|
||
|
|
"name": "mmlu",
|
||
|
|
"sample_count": {
|
||
|
|
"acc,none": 14042
|
||
|
|
},
|
||
|
|
"sample_len": 14042
|
||
|
|
},
|
||
|
|
"mmlu_humanities": {
|
||
|
|
"acc,none": 0.5596174282678003,
|
||
|
|
"acc_stderr,none": 0.00661316692569494,
|
||
|
|
"alias": "humanities",
|
||
|
|
"name": "mmlu_humanities",
|
||
|
|
"sample_count": {
|
||
|
|
"acc,none": 4705
|
||
|
|
},
|
||
|
|
"sample_len": 4705
|
||
|
|
},
|
||
|
|
"mmlu_other": {
|
||
|
|
"acc,none": 0.6903765690376569,
|
||
|
|
"acc_stderr,none": 0.00793504827765504,
|
||
|
|
"alias": "other",
|
||
|
|
"name": "mmlu_other",
|
||
|
|
"sample_count": {
|
||
|
|
"acc,none": 3107
|
||
|
|
},
|
||
|
|
"sample_len": 3107
|
||
|
|
},
|
||
|
|
"mmlu_social_sciences": {
|
||
|
|
"acc,none": 0.7565810854728632,
|
||
|
|
"acc_stderr,none": 0.007580062219729006,
|
||
|
|
"alias": "social sciences",
|
||
|
|
"name": "mmlu_social_sciences",
|
||
|
|
"sample_count": {
|
||
|
|
"acc,none": 3077
|
||
|
|
},
|
||
|
|
"sample_len": 3077
|
||
|
|
},
|
||
|
|
"mmlu_stem": {
|
||
|
|
"acc,none": 0.5026958452267681,
|
||
|
|
"acc_stderr,none": 0.008068176972734112,
|
||
|
|
"alias": "stem",
|
||
|
|
"name": "mmlu_stem",
|
||
|
|
"sample_count": {
|
||
|
|
"acc,none": 3153
|
||
|
|
},
|
||
|
|
"sample_len": 3153
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"higher_is_better": {
|
||
|
|
"mmlu_abstract_algebra": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"mmlu_anatomy": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"mmlu_astronomy": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"mmlu_business_ethics": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"mmlu_clinical_knowledge": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"mmlu_college_biology": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"mmlu_college_chemistry": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"mmlu_college_computer_science": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"mmlu_college_mathematics": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"mmlu_college_medicine": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"mmlu_college_physics": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"mmlu_computer_security": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"mmlu_conceptual_physics": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"mmlu_econometrics": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"mmlu_electrical_engineering": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"mmlu_elementary_mathematics": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"mmlu_formal_logic": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"mmlu_global_facts": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"mmlu_high_school_biology": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"mmlu_high_school_chemistry": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"mmlu_high_school_computer_science": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"mmlu_high_school_european_history": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"mmlu_high_school_geography": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"mmlu_high_school_government_and_politics": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"mmlu_high_school_macroeconomics": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"mmlu_high_school_mathematics": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"mmlu_high_school_microeconomics": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"mmlu_high_school_physics": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"mmlu_high_school_psychology": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"mmlu_high_school_statistics": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"mmlu_high_school_us_history": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"mmlu_high_school_world_history": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"mmlu_human_aging": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"mmlu_human_sexuality": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"mmlu_humanities": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"mmlu_international_law": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"mmlu_jurisprudence": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"mmlu_logical_fallacies": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"mmlu_machine_learning": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"mmlu_management": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"mmlu_marketing": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"mmlu_medical_genetics": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"mmlu_miscellaneous": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"mmlu_moral_disputes": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"mmlu_moral_scenarios": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"mmlu_nutrition": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"mmlu_other": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"mmlu_philosophy": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"mmlu_prehistory": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"mmlu_professional_accounting": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"mmlu_professional_law": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"mmlu_professional_medicine": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"mmlu_professional_psychology": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"mmlu_public_relations": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"mmlu_security_studies": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"mmlu_social_sciences": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"mmlu_sociology": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"mmlu_stem": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"mmlu_us_foreign_policy": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"mmlu_virology": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"mmlu_world_religions": {
|
||
|
|
"acc": true
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"lm_eval_version": "0.4.12",
|
||
|
|
"max_length": 262144,
|
||
|
|
"model_name": "Qwen/Qwen3-4B-Instruct-2507",
|
||
|
|
"model_name_sanitized": "Qwen/Qwen3-4B-Instruct-2507",
|
||
|
|
"model_source": "Qwen/Qwen3-4B-Instruct-2507",
|
||
|
|
"n-samples": {
|
||
|
|
"mmlu_abstract_algebra": {
|
||
|
|
"effective": 100,
|
||
|
|
"original": 100
|
||
|
|
},
|
||
|
|
"mmlu_anatomy": {
|
||
|
|
"effective": 135,
|
||
|
|
"original": 135
|
||
|
|
},
|
||
|
|
"mmlu_astronomy": {
|
||
|
|
"effective": 152,
|
||
|
|
"original": 152
|
||
|
|
},
|
||
|
|
"mmlu_business_ethics": {
|
||
|
|
"effective": 100,
|
||
|
|
"original": 100
|
||
|
|
},
|
||
|
|
"mmlu_clinical_knowledge": {
|
||
|
|
"effective": 265,
|
||
|
|
"original": 265
|
||
|
|
},
|
||
|
|
"mmlu_college_biology": {
|
||
|
|
"effective": 144,
|
||
|
|
"original": 144
|
||
|
|
},
|
||
|
|
"mmlu_college_chemistry": {
|
||
|
|
"effective": 100,
|
||
|
|
"original": 100
|
||
|
|
},
|
||
|
|
"mmlu_college_computer_science": {
|
||
|
|
"effective": 100,
|
||
|
|
"original": 100
|
||
|
|
},
|
||
|
|
"mmlu_college_mathematics": {
|
||
|
|
"effective": 100,
|
||
|
|
"original": 100
|
||
|
|
},
|
||
|
|
"mmlu_college_medicine": {
|
||
|
|
"effective": 173,
|
||
|
|
"original": 173
|
||
|
|
},
|
||
|
|
"mmlu_college_physics": {
|
||
|
|
"effective": 102,
|
||
|
|
"original": 102
|
||
|
|
},
|
||
|
|
"mmlu_computer_security": {
|
||
|
|
"effective": 100,
|
||
|
|
"original": 100
|
||
|
|
},
|
||
|
|
"mmlu_conceptual_physics": {
|
||
|
|
"effective": 235,
|
||
|
|
"original": 235
|
||
|
|
},
|
||
|
|
"mmlu_econometrics": {
|
||
|
|
"effective": 114,
|
||
|
|
"original": 114
|
||
|
|
},
|
||
|
|
"mmlu_electrical_engineering": {
|
||
|
|
"effective": 145,
|
||
|
|
"original": 145
|
||
|
|
},
|
||
|
|
"mmlu_elementary_mathematics": {
|
||
|
|
"effective": 378,
|
||
|
|
"original": 378
|
||
|
|
},
|
||
|
|
"mmlu_formal_logic": {
|
||
|
|
"effective": 126,
|
||
|
|
"original": 126
|
||
|
|
},
|
||
|
|
"mmlu_global_facts": {
|
||
|
|
"effective": 100,
|
||
|
|
"original": 100
|
||
|
|
},
|
||
|
|
"mmlu_high_school_biology": {
|
||
|
|
"effective": 310,
|
||
|
|
"original": 310
|
||
|
|
},
|
||
|
|
"mmlu_high_school_chemistry": {
|
||
|
|
"effective": 203,
|
||
|
|
"original": 203
|
||
|
|
},
|
||
|
|
"mmlu_high_school_computer_science": {
|
||
|
|
"effective": 100,
|
||
|
|
"original": 100
|
||
|
|
},
|
||
|
|
"mmlu_high_school_european_history": {
|
||
|
|
"effective": 165,
|
||
|
|
"original": 165
|
||
|
|
},
|
||
|
|
"mmlu_high_school_geography": {
|
||
|
|
"effective": 198,
|
||
|
|
"original": 198
|
||
|
|
},
|
||
|
|
"mmlu_high_school_government_and_politics": {
|
||
|
|
"effective": 193,
|
||
|
|
"original": 193
|
||
|
|
},
|
||
|
|
"mmlu_high_school_macroeconomics": {
|
||
|
|
"effective": 390,
|
||
|
|
"original": 390
|
||
|
|
},
|
||
|
|
"mmlu_high_school_mathematics": {
|
||
|
|
"effective": 270,
|
||
|
|
"original": 270
|
||
|
|
},
|
||
|
|
"mmlu_high_school_microeconomics": {
|
||
|
|
"effective": 238,
|
||
|
|
"original": 238
|
||
|
|
},
|
||
|
|
"mmlu_high_school_physics": {
|
||
|
|
"effective": 151,
|
||
|
|
"original": 151
|
||
|
|
},
|
||
|
|
"mmlu_high_school_psychology": {
|
||
|
|
"effective": 545,
|
||
|
|
"original": 545
|
||
|
|
},
|
||
|
|
"mmlu_high_school_statistics": {
|
||
|
|
"effective": 216,
|
||
|
|
"original": 216
|
||
|
|
},
|
||
|
|
"mmlu_high_school_us_history": {
|
||
|
|
"effective": 204,
|
||
|
|
"original": 204
|
||
|
|
},
|
||
|
|
"mmlu_high_school_world_history": {
|
||
|
|
"effective": 237,
|
||
|
|
"original": 237
|
||
|
|
},
|
||
|
|
"mmlu_human_aging": {
|
||
|
|
"effective": 223,
|
||
|
|
"original": 223
|
||
|
|
},
|
||
|
|
"mmlu_human_sexuality": {
|
||
|
|
"effective": 131,
|
||
|
|
"original": 131
|
||
|
|
},
|
||
|
|
"mmlu_international_law": {
|
||
|
|
"effective": 121,
|
||
|
|
"original": 121
|
||
|
|
},
|
||
|
|
"mmlu_jurisprudence": {
|
||
|
|
"effective": 108,
|
||
|
|
"original": 108
|
||
|
|
},
|
||
|
|
"mmlu_logical_fallacies": {
|
||
|
|
"effective": 163,
|
||
|
|
"original": 163
|
||
|
|
},
|
||
|
|
"mmlu_machine_learning": {
|
||
|
|
"effective": 112,
|
||
|
|
"original": 112
|
||
|
|
},
|
||
|
|
"mmlu_management": {
|
||
|
|
"effective": 103,
|
||
|
|
"original": 103
|
||
|
|
},
|
||
|
|
"mmlu_marketing": {
|
||
|
|
"effective": 234,
|
||
|
|
"original": 234
|
||
|
|
},
|
||
|
|
"mmlu_medical_genetics": {
|
||
|
|
"effective": 100,
|
||
|
|
"original": 100
|
||
|
|
},
|
||
|
|
"mmlu_miscellaneous": {
|
||
|
|
"effective": 783,
|
||
|
|
"original": 783
|
||
|
|
},
|
||
|
|
"mmlu_moral_disputes": {
|
||
|
|
"effective": 346,
|
||
|
|
"original": 346
|
||
|
|
},
|
||
|
|
"mmlu_moral_scenarios": {
|
||
|
|
"effective": 895,
|
||
|
|
"original": 895
|
||
|
|
},
|
||
|
|
"mmlu_nutrition": {
|
||
|
|
"effective": 306,
|
||
|
|
"original": 306
|
||
|
|
},
|
||
|
|
"mmlu_philosophy": {
|
||
|
|
"effective": 311,
|
||
|
|
"original": 311
|
||
|
|
},
|
||
|
|
"mmlu_prehistory": {
|
||
|
|
"effective": 324,
|
||
|
|
"original": 324
|
||
|
|
},
|
||
|
|
"mmlu_professional_accounting": {
|
||
|
|
"effective": 282,
|
||
|
|
"original": 282
|
||
|
|
},
|
||
|
|
"mmlu_professional_law": {
|
||
|
|
"effective": 1534,
|
||
|
|
"original": 1534
|
||
|
|
},
|
||
|
|
"mmlu_professional_medicine": {
|
||
|
|
"effective": 272,
|
||
|
|
"original": 272
|
||
|
|
},
|
||
|
|
"mmlu_professional_psychology": {
|
||
|
|
"effective": 612,
|
||
|
|
"original": 612
|
||
|
|
},
|
||
|
|
"mmlu_public_relations": {
|
||
|
|
"effective": 110,
|
||
|
|
"original": 110
|
||
|
|
},
|
||
|
|
"mmlu_security_studies": {
|
||
|
|
"effective": 245,
|
||
|
|
"original": 245
|
||
|
|
},
|
||
|
|
"mmlu_sociology": {
|
||
|
|
"effective": 201,
|
||
|
|
"original": 201
|
||
|
|
},
|
||
|
|
"mmlu_us_foreign_policy": {
|
||
|
|
"effective": 100,
|
||
|
|
"original": 100
|
||
|
|
},
|
||
|
|
"mmlu_virology": {
|
||
|
|
"effective": 166,
|
||
|
|
"original": 166
|
||
|
|
},
|
||
|
|
"mmlu_world_religions": {
|
||
|
|
"effective": 171,
|
||
|
|
"original": 171
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"n-shot": {
|
||
|
|
"mmlu_abstract_algebra": 0,
|
||
|
|
"mmlu_anatomy": 0,
|
||
|
|
"mmlu_astronomy": 0,
|
||
|
|
"mmlu_business_ethics": 0,
|
||
|
|
"mmlu_clinical_knowledge": 0,
|
||
|
|
"mmlu_college_biology": 0,
|
||
|
|
"mmlu_college_chemistry": 0,
|
||
|
|
"mmlu_college_computer_science": 0,
|
||
|
|
"mmlu_college_mathematics": 0,
|
||
|
|
"mmlu_college_medicine": 0,
|
||
|
|
"mmlu_college_physics": 0,
|
||
|
|
"mmlu_computer_security": 0,
|
||
|
|
"mmlu_conceptual_physics": 0,
|
||
|
|
"mmlu_econometrics": 0,
|
||
|
|
"mmlu_electrical_engineering": 0,
|
||
|
|
"mmlu_elementary_mathematics": 0,
|
||
|
|
"mmlu_formal_logic": 0,
|
||
|
|
"mmlu_global_facts": 0,
|
||
|
|
"mmlu_high_school_biology": 0,
|
||
|
|
"mmlu_high_school_chemistry": 0,
|
||
|
|
"mmlu_high_school_computer_science": 0,
|
||
|
|
"mmlu_high_school_european_history": 0,
|
||
|
|
"mmlu_high_school_geography": 0,
|
||
|
|
"mmlu_high_school_government_and_politics": 0,
|
||
|
|
"mmlu_high_school_macroeconomics": 0,
|
||
|
|
"mmlu_high_school_mathematics": 0,
|
||
|
|
"mmlu_high_school_microeconomics": 0,
|
||
|
|
"mmlu_high_school_physics": 0,
|
||
|
|
"mmlu_high_school_psychology": 0,
|
||
|
|
"mmlu_high_school_statistics": 0,
|
||
|
|
"mmlu_high_school_us_history": 0,
|
||
|
|
"mmlu_high_school_world_history": 0,
|
||
|
|
"mmlu_human_aging": 0,
|
||
|
|
"mmlu_human_sexuality": 0,
|
||
|
|
"mmlu_humanities": 0,
|
||
|
|
"mmlu_international_law": 0,
|
||
|
|
"mmlu_jurisprudence": 0,
|
||
|
|
"mmlu_logical_fallacies": 0,
|
||
|
|
"mmlu_machine_learning": 0,
|
||
|
|
"mmlu_management": 0,
|
||
|
|
"mmlu_marketing": 0,
|
||
|
|
"mmlu_medical_genetics": 0,
|
||
|
|
"mmlu_miscellaneous": 0,
|
||
|
|
"mmlu_moral_disputes": 0,
|
||
|
|
"mmlu_moral_scenarios": 0,
|
||
|
|
"mmlu_nutrition": 0,
|
||
|
|
"mmlu_other": 0,
|
||
|
|
"mmlu_philosophy": 0,
|
||
|
|
"mmlu_prehistory": 0,
|
||
|
|
"mmlu_professional_accounting": 0,
|
||
|
|
"mmlu_professional_law": 0,
|
||
|
|
"mmlu_professional_medicine": 0,
|
||
|
|
"mmlu_professional_psychology": 0,
|
||
|
|
"mmlu_public_relations": 0,
|
||
|
|
"mmlu_security_studies": 0,
|
||
|
|
"mmlu_social_sciences": 0,
|
||
|
|
"mmlu_sociology": 0,
|
||
|
|
"mmlu_stem": 0,
|
||
|
|
"mmlu_us_foreign_policy": 0,
|
||
|
|
"mmlu_virology": 0,
|
||
|
|
"mmlu_world_religions": 0
|
||
|
|
},
|
||
|
|
"results": {
|
||
|
|
"mmlu": {
|
||
|
|
"acc,none": 0.6189289275032047,
|
||
|
|
"acc_stderr,none": 0.003746138843326914,
|
||
|
|
"alias": "mmlu",
|
||
|
|
"name": "mmlu",
|
||
|
|
"sample_count": {
|
||
|
|
"acc,none": 14042
|
||
|
|
},
|
||
|
|
"sample_len": 14042
|
||
|
|
},
|
||
|
|
"mmlu_abstract_algebra": {
|
||
|
|
"acc,none": 0.23,
|
||
|
|
"acc_stderr,none": 0.04229525846816507,
|
||
|
|
"alias": "abstract_algebra",
|
||
|
|
"name": "mmlu_abstract_algebra",
|
||
|
|
"sample_len": 100
|
||
|
|
},
|
||
|
|
"mmlu_anatomy": {
|
||
|
|
"acc,none": 0.6444444444444445,
|
||
|
|
"acc_stderr,none": 0.04135176749720385,
|
||
|
|
"alias": "anatomy",
|
||
|
|
"name": "mmlu_anatomy",
|
||
|
|
"sample_len": 135
|
||
|
|
},
|
||
|
|
"mmlu_astronomy": {
|
||
|
|
"acc,none": 0.7828947368421053,
|
||
|
|
"acc_stderr,none": 0.03355045304882924,
|
||
|
|
"alias": "astronomy",
|
||
|
|
"name": "mmlu_astronomy",
|
||
|
|
"sample_len": 152
|
||
|
|
},
|
||
|
|
"mmlu_business_ethics": {
|
||
|
|
"acc,none": 0.7,
|
||
|
|
"acc_stderr,none": 0.04605661864718383,
|
||
|
|
"alias": "business_ethics",
|
||
|
|
"name": "mmlu_business_ethics",
|
||
|
|
"sample_len": 100
|
||
|
|
},
|
||
|
|
"mmlu_clinical_knowledge": {
|
||
|
|
"acc,none": 0.7584905660377359,
|
||
|
|
"acc_stderr,none": 0.02634148037111831,
|
||
|
|
"alias": "clinical_knowledge",
|
||
|
|
"name": "mmlu_clinical_knowledge",
|
||
|
|
"sample_len": 265
|
||
|
|
},
|
||
|
|
"mmlu_college_biology": {
|
||
|
|
"acc,none": 0.8194444444444444,
|
||
|
|
"acc_stderr,none": 0.03216600808802272,
|
||
|
|
"alias": "college_biology",
|
||
|
|
"name": "mmlu_college_biology",
|
||
|
|
"sample_len": 144
|
||
|
|
},
|
||
|
|
"mmlu_college_chemistry": {
|
||
|
|
"acc,none": 0.42,
|
||
|
|
"acc_stderr,none": 0.04960449637488583,
|
||
|
|
"alias": "college_chemistry",
|
||
|
|
"name": "mmlu_college_chemistry",
|
||
|
|
"sample_len": 100
|
||
|
|
},
|
||
|
|
"mmlu_college_computer_science": {
|
||
|
|
"acc,none": 0.5,
|
||
|
|
"acc_stderr,none": 0.050251890762960605,
|
||
|
|
"alias": "college_computer_science",
|
||
|
|
"name": "mmlu_college_computer_science",
|
||
|
|
"sample_len": 100
|
||
|
|
},
|
||
|
|
"mmlu_college_mathematics": {
|
||
|
|
"acc,none": 0.22,
|
||
|
|
"acc_stderr,none": 0.041633319989322654,
|
||
|
|
"alias": "college_mathematics",
|
||
|
|
"name": "mmlu_college_mathematics",
|
||
|
|
"sample_len": 100
|
||
|
|
},
|
||
|
|
"mmlu_college_medicine": {
|
||
|
|
"acc,none": 0.6358381502890174,
|
||
|
|
"acc_stderr,none": 0.03669072477416912,
|
||
|
|
"alias": "college_medicine",
|
||
|
|
"name": "mmlu_college_medicine",
|
||
|
|
"sample_len": 173
|
||
|
|
},
|
||
|
|
"mmlu_college_physics": {
|
||
|
|
"acc,none": 0.39215686274509803,
|
||
|
|
"acc_stderr,none": 0.04858083574266346,
|
||
|
|
"alias": "college_physics",
|
||
|
|
"name": "mmlu_college_physics",
|
||
|
|
"sample_len": 102
|
||
|
|
},
|
||
|
|
"mmlu_computer_security": {
|
||
|
|
"acc,none": 0.7,
|
||
|
|
"acc_stderr,none": 0.04605661864718383,
|
||
|
|
"alias": "computer_security",
|
||
|
|
"name": "mmlu_computer_security",
|
||
|
|
"sample_len": 100
|
||
|
|
},
|
||
|
|
"mmlu_conceptual_physics": {
|
||
|
|
"acc,none": 0.6808510638297872,
|
||
|
|
"acc_stderr,none": 0.03047297336338012,
|
||
|
|
"alias": "conceptual_physics",
|
||
|
|
"name": "mmlu_conceptual_physics",
|
||
|
|
"sample_len": 235
|
||
|
|
},
|
||
|
|
"mmlu_econometrics": {
|
||
|
|
"acc,none": 0.5175438596491229,
|
||
|
|
"acc_stderr,none": 0.04700708033551044,
|
||
|
|
"alias": "econometrics",
|
||
|
|
"name": "mmlu_econometrics",
|
||
|
|
"sample_len": 114
|
||
|
|
},
|
||
|
|
"mmlu_electrical_engineering": {
|
||
|
|
"acc,none": 0.593103448275862,
|
||
|
|
"acc_stderr,none": 0.040937939812662354,
|
||
|
|
"alias": "electrical_engineering",
|
||
|
|
"name": "mmlu_electrical_engineering",
|
||
|
|
"sample_len": 145
|
||
|
|
},
|
||
|
|
"mmlu_elementary_mathematics": {
|
||
|
|
"acc,none": 0.24338624338624337,
|
||
|
|
"acc_stderr,none": 0.022101128787415353,
|
||
|
|
"alias": "elementary_mathematics",
|
||
|
|
"name": "mmlu_elementary_mathematics",
|
||
|
|
"sample_len": 378
|
||
|
|
},
|
||
|
|
"mmlu_formal_logic": {
|
||
|
|
"acc,none": 0.3333333333333333,
|
||
|
|
"acc_stderr,none": 0.04216370213557839,
|
||
|
|
"alias": "formal_logic",
|
||
|
|
"name": "mmlu_formal_logic",
|
||
|
|
"sample_len": 126
|
||
|
|
},
|
||
|
|
"mmlu_global_facts": {
|
||
|
|
"acc,none": 0.38,
|
||
|
|
"acc_stderr,none": 0.04878317312145634,
|
||
|
|
"alias": "global_facts",
|
||
|
|
"name": "mmlu_global_facts",
|
||
|
|
"sample_len": 100
|
||
|
|
},
|
||
|
|
"mmlu_high_school_biology": {
|
||
|
|
"acc,none": 0.8161290322580645,
|
||
|
|
"acc_stderr,none": 0.022037217340267787,
|
||
|
|
"alias": "high_school_biology",
|
||
|
|
"name": "mmlu_high_school_biology",
|
||
|
|
"sample_len": 310
|
||
|
|
},
|
||
|
|
"mmlu_high_school_chemistry": {
|
||
|
|
"acc,none": 0.458128078817734,
|
||
|
|
"acc_stderr,none": 0.035056301407857406,
|
||
|
|
"alias": "high_school_chemistry",
|
||
|
|
"name": "mmlu_high_school_chemistry",
|
||
|
|
"sample_len": 203
|
||
|
|
},
|
||
|
|
"mmlu_high_school_computer_science": {
|
||
|
|
"acc,none": 0.69,
|
||
|
|
"acc_stderr,none": 0.046482319871173176,
|
||
|
|
"alias": "high_school_computer_science",
|
||
|
|
"name": "mmlu_high_school_computer_science",
|
||
|
|
"sample_len": 100
|
||
|
|
},
|
||
|
|
"mmlu_high_school_european_history": {
|
||
|
|
"acc,none": 0.8,
|
||
|
|
"acc_stderr,none": 0.031234752377721213,
|
||
|
|
"alias": "high_school_european_history",
|
||
|
|
"name": "mmlu_high_school_european_history",
|
||
|
|
"sample_len": 165
|
||
|
|
},
|
||
|
|
"mmlu_high_school_geography": {
|
||
|
|
"acc,none": 0.7929292929292929,
|
||
|
|
"acc_stderr,none": 0.02886977846026699,
|
||
|
|
"alias": "high_school_geography",
|
||
|
|
"name": "mmlu_high_school_geography",
|
||
|
|
"sample_len": 198
|
||
|
|
},
|
||
|
|
"mmlu_high_school_government_and_politics": {
|
||
|
|
"acc,none": 0.8756476683937824,
|
||
|
|
"acc_stderr,none": 0.02381447708659357,
|
||
|
|
"alias": "high_school_government_and_politics",
|
||
|
|
"name": "mmlu_high_school_government_and_politics",
|
||
|
|
"sample_len": 193
|
||
|
|
},
|
||
|
|
"mmlu_high_school_macroeconomics": {
|
||
|
|
"acc,none": 0.6615384615384615,
|
||
|
|
"acc_stderr,none": 0.023991500500313064,
|
||
|
|
"alias": "high_school_macroeconomics",
|
||
|
|
"name": "mmlu_high_school_macroeconomics",
|
||
|
|
"sample_len": 390
|
||
|
|
},
|
||
|
|
"mmlu_high_school_mathematics": {
|
||
|
|
"acc,none": 0.2111111111111111,
|
||
|
|
"acc_stderr,none": 0.02488211685765511,
|
||
|
|
"alias": "high_school_mathematics",
|
||
|
|
"name": "mmlu_high_school_mathematics",
|
||
|
|
"sample_len": 270
|
||
|
|
},
|
||
|
|
"mmlu_high_school_microeconomics": {
|
||
|
|
"acc,none": 0.7857142857142857,
|
||
|
|
"acc_stderr,none": 0.026653531596715505,
|
||
|
|
"alias": "high_school_microeconomics",
|
||
|
|
"name": "mmlu_high_school_microeconomics",
|
||
|
|
"sample_len": 238
|
||
|
|
},
|
||
|
|
"mmlu_high_school_physics": {
|
||
|
|
"acc,none": 0.3841059602649007,
|
||
|
|
"acc_stderr,none": 0.03971301814719192,
|
||
|
|
"alias": "high_school_physics",
|
||
|
|
"name": "mmlu_high_school_physics",
|
||
|
|
"sample_len": 151
|
||
|
|
},
|
||
|
|
"mmlu_high_school_psychology": {
|
||
|
|
"acc,none": 0.8660550458715597,
|
||
|
|
"acc_stderr,none": 0.014602811435592633,
|
||
|
|
"alias": "high_school_psychology",
|
||
|
|
"name": "mmlu_high_school_psychology",
|
||
|
|
"sample_len": 545
|
||
|
|
},
|
||
|
|
"mmlu_high_school_statistics": {
|
||
|
|
"acc,none": 0.4537037037037037,
|
||
|
|
"acc_stderr,none": 0.033953227263757976,
|
||
|
|
"alias": "high_school_statistics",
|
||
|
|
"name": "mmlu_high_school_statistics",
|
||
|
|
"sample_len": 216
|
||
|
|
},
|
||
|
|
"mmlu_high_school_us_history": {
|
||
|
|
"acc,none": 0.8284313725490197,
|
||
|
|
"acc_stderr,none": 0.026460569561240675,
|
||
|
|
"alias": "high_school_us_history",
|
||
|
|
"name": "mmlu_high_school_us_history",
|
||
|
|
"sample_len": 204
|
||
|
|
},
|
||
|
|
"mmlu_high_school_world_history": {
|
||
|
|
"acc,none": 0.8270042194092827,
|
||
|
|
"acc_stderr,none": 0.024621562866768382,
|
||
|
|
"alias": "high_school_world_history",
|
||
|
|
"name": "mmlu_high_school_world_history",
|
||
|
|
"sample_len": 237
|
||
|
|
},
|
||
|
|
"mmlu_human_aging": {
|
||
|
|
"acc,none": 0.6367713004484304,
|
||
|
|
"acc_stderr,none": 0.03227790442850494,
|
||
|
|
"alias": "human_aging",
|
||
|
|
"name": "mmlu_human_aging",
|
||
|
|
"sample_len": 223
|
||
|
|
},
|
||
|
|
"mmlu_human_sexuality": {
|
||
|
|
"acc,none": 0.732824427480916,
|
||
|
|
"acc_stderr,none": 0.03880848301082397,
|
||
|
|
"alias": "human_sexuality",
|
||
|
|
"name": "mmlu_human_sexuality",
|
||
|
|
"sample_len": 131
|
||
|
|
},
|
||
|
|
"mmlu_humanities": {
|
||
|
|
"acc,none": 0.5596174282678003,
|
||
|
|
"acc_stderr,none": 0.00661316692569494,
|
||
|
|
"alias": "humanities",
|
||
|
|
"name": "mmlu_humanities",
|
||
|
|
"sample_count": {
|
||
|
|
"acc,none": 4705
|
||
|
|
},
|
||
|
|
"sample_len": 4705
|
||
|
|
},
|
||
|
|
"mmlu_international_law": {
|
||
|
|
"acc,none": 0.8347107438016529,
|
||
|
|
"acc_stderr,none": 0.03390780612972774,
|
||
|
|
"alias": "international_law",
|
||
|
|
"name": "mmlu_international_law",
|
||
|
|
"sample_len": 121
|
||
|
|
},
|
||
|
|
"mmlu_jurisprudence": {
|
||
|
|
"acc,none": 0.7407407407407407,
|
||
|
|
"acc_stderr,none": 0.04236511258094632,
|
||
|
|
"alias": "jurisprudence",
|
||
|
|
"name": "mmlu_jurisprudence",
|
||
|
|
"sample_len": 108
|
||
|
|
},
|
||
|
|
"mmlu_logical_fallacies": {
|
||
|
|
"acc,none": 0.7116564417177914,
|
||
|
|
"acc_stderr,none": 0.03559039531617345,
|
||
|
|
"alias": "logical_fallacies",
|
||
|
|
"name": "mmlu_logical_fallacies",
|
||
|
|
"sample_len": 163
|
||
|
|
},
|
||
|
|
"mmlu_machine_learning": {
|
||
|
|
"acc,none": 0.42857142857142855,
|
||
|
|
"acc_stderr,none": 0.04697113923010208,
|
||
|
|
"alias": "machine_learning",
|
||
|
|
"name": "mmlu_machine_learning",
|
||
|
|
"sample_len": 112
|
||
|
|
},
|
||
|
|
"mmlu_management": {
|
||
|
|
"acc,none": 0.8543689320388349,
|
||
|
|
"acc_stderr,none": 0.034926064766237934,
|
||
|
|
"alias": "management",
|
||
|
|
"name": "mmlu_management",
|
||
|
|
"sample_len": 103
|
||
|
|
},
|
||
|
|
"mmlu_marketing": {
|
||
|
|
"acc,none": 0.8461538461538461,
|
||
|
|
"acc_stderr,none": 0.023636873317489267,
|
||
|
|
"alias": "marketing",
|
||
|
|
"name": "mmlu_marketing",
|
||
|
|
"sample_len": 234
|
||
|
|
},
|
||
|
|
"mmlu_medical_genetics": {
|
||
|
|
"acc,none": 0.73,
|
||
|
|
"acc_stderr,none": 0.04461960433384737,
|
||
|
|
"alias": "medical_genetics",
|
||
|
|
"name": "mmlu_medical_genetics",
|
||
|
|
"sample_len": 100
|
||
|
|
},
|
||
|
|
"mmlu_miscellaneous": {
|
||
|
|
"acc,none": 0.8033205619412516,
|
||
|
|
"acc_stderr,none": 0.014214138556913851,
|
||
|
|
"alias": "miscellaneous",
|
||
|
|
"name": "mmlu_miscellaneous",
|
||
|
|
"sample_len": 783
|
||
|
|
},
|
||
|
|
"mmlu_moral_disputes": {
|
||
|
|
"acc,none": 0.7138728323699421,
|
||
|
|
"acc_stderr,none": 0.02433214677913417,
|
||
|
|
"alias": "moral_disputes",
|
||
|
|
"name": "mmlu_moral_disputes",
|
||
|
|
"sample_len": 346
|
||
|
|
},
|
||
|
|
"mmlu_moral_scenarios": {
|
||
|
|
"acc,none": 0.26927374301675977,
|
||
|
|
"acc_stderr,none": 0.014835616582882544,
|
||
|
|
"alias": "moral_scenarios",
|
||
|
|
"name": "mmlu_moral_scenarios",
|
||
|
|
"sample_len": 895
|
||
|
|
},
|
||
|
|
"mmlu_nutrition": {
|
||
|
|
"acc,none": 0.7647058823529411,
|
||
|
|
"acc_stderr,none": 0.02428861946604616,
|
||
|
|
"alias": "nutrition",
|
||
|
|
"name": "mmlu_nutrition",
|
||
|
|
"sample_len": 306
|
||
|
|
},
|
||
|
|
"mmlu_other": {
|
||
|
|
"acc,none": 0.6903765690376569,
|
||
|
|
"acc_stderr,none": 0.00793504827765504,
|
||
|
|
"alias": "other",
|
||
|
|
"name": "mmlu_other",
|
||
|
|
"sample_count": {
|
||
|
|
"acc,none": 3107
|
||
|
|
},
|
||
|
|
"sample_len": 3107
|
||
|
|
},
|
||
|
|
"mmlu_philosophy": {
|
||
|
|
"acc,none": 0.7041800643086816,
|
||
|
|
"acc_stderr,none": 0.025922371788818833,
|
||
|
|
"alias": "philosophy",
|
||
|
|
"name": "mmlu_philosophy",
|
||
|
|
"sample_len": 311
|
||
|
|
},
|
||
|
|
"mmlu_prehistory": {
|
||
|
|
"acc,none": 0.7654320987654321,
|
||
|
|
"acc_stderr,none": 0.0235768817440057,
|
||
|
|
"alias": "prehistory",
|
||
|
|
"name": "mmlu_prehistory",
|
||
|
|
"sample_len": 324
|
||
|
|
},
|
||
|
|
"mmlu_professional_accounting": {
|
||
|
|
"acc,none": 0.46808510638297873,
|
||
|
|
"acc_stderr,none": 0.02976667507587383,
|
||
|
|
"alias": "professional_accounting",
|
||
|
|
"name": "mmlu_professional_accounting",
|
||
|
|
"sample_len": 282
|
||
|
|
},
|
||
|
|
"mmlu_professional_law": {
|
||
|
|
"acc,none": 0.4589308996088657,
|
||
|
|
"acc_stderr,none": 0.0127270848267999,
|
||
|
|
"alias": "professional_law",
|
||
|
|
"name": "mmlu_professional_law",
|
||
|
|
"sample_len": 1534
|
||
|
|
},
|
||
|
|
"mmlu_professional_medicine": {
|
||
|
|
"acc,none": 0.5551470588235294,
|
||
|
|
"acc_stderr,none": 0.030187532060329314,
|
||
|
|
"alias": "professional_medicine",
|
||
|
|
"name": "mmlu_professional_medicine",
|
||
|
|
"sample_len": 272
|
||
|
|
},
|
||
|
|
"mmlu_professional_psychology": {
|
||
|
|
"acc,none": 0.6993464052287581,
|
||
|
|
"acc_stderr,none": 0.01855063450295291,
|
||
|
|
"alias": "professional_psychology",
|
||
|
|
"name": "mmlu_professional_psychology",
|
||
|
|
"sample_len": 612
|
||
|
|
},
|
||
|
|
"mmlu_public_relations": {
|
||
|
|
"acc,none": 0.6454545454545455,
|
||
|
|
"acc_stderr,none": 0.04582004841505413,
|
||
|
|
"alias": "public_relations",
|
||
|
|
"name": "mmlu_public_relations",
|
||
|
|
"sample_len": 110
|
||
|
|
},
|
||
|
|
"mmlu_security_studies": {
|
||
|
|
"acc,none": 0.7428571428571429,
|
||
|
|
"acc_stderr,none": 0.027979823538744557,
|
||
|
|
"alias": "security_studies",
|
||
|
|
"name": "mmlu_security_studies",
|
||
|
|
"sample_len": 245
|
||
|
|
},
|
||
|
|
"mmlu_social_sciences": {
|
||
|
|
"acc,none": 0.7565810854728632,
|
||
|
|
"acc_stderr,none": 0.007580062219729006,
|
||
|
|
"alias": "social sciences",
|
||
|
|
"name": "mmlu_social_sciences",
|
||
|
|
"sample_count": {
|
||
|
|
"acc,none": 3077
|
||
|
|
},
|
||
|
|
"sample_len": 3077
|
||
|
|
},
|
||
|
|
"mmlu_sociology": {
|
||
|
|
"acc,none": 0.835820895522388,
|
||
|
|
"acc_stderr,none": 0.026193923544454094,
|
||
|
|
"alias": "sociology",
|
||
|
|
"name": "mmlu_sociology",
|
||
|
|
"sample_len": 201
|
||
|
|
},
|
||
|
|
"mmlu_stem": {
|
||
|
|
"acc,none": 0.5026958452267681,
|
||
|
|
"acc_stderr,none": 0.008068176972734112,
|
||
|
|
"alias": "stem",
|
||
|
|
"name": "mmlu_stem",
|
||
|
|
"sample_count": {
|
||
|
|
"acc,none": 3153
|
||
|
|
},
|
||
|
|
"sample_len": 3153
|
||
|
|
},
|
||
|
|
"mmlu_us_foreign_policy": {
|
||
|
|
"acc,none": 0.81,
|
||
|
|
"acc_stderr,none": 0.039427724440366255,
|
||
|
|
"alias": "us_foreign_policy",
|
||
|
|
"name": "mmlu_us_foreign_policy",
|
||
|
|
"sample_len": 100
|
||
|
|
},
|
||
|
|
"mmlu_virology": {
|
||
|
|
"acc,none": 0.4759036144578313,
|
||
|
|
"acc_stderr,none": 0.03887971849597268,
|
||
|
|
"alias": "virology",
|
||
|
|
"name": "mmlu_virology",
|
||
|
|
"sample_len": 166
|
||
|
|
},
|
||
|
|
"mmlu_world_religions": {
|
||
|
|
"acc,none": 0.8070175438596491,
|
||
|
|
"acc_stderr,none": 0.030267457554898448,
|
||
|
|
"alias": "world_religions",
|
||
|
|
"name": "mmlu_world_religions",
|
||
|
|
"sample_len": 171
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"system_instruction": null,
|
||
|
|
"system_instruction_sha": null,
|
||
|
|
"task_hashes": {
|
||
|
|
"mmlu_abstract_algebra": "1ecc29cafeff483bf80347adc2878dae00d01596988032f438afce6291c8cfe7",
|
||
|
|
"mmlu_anatomy": "51232e78f32967e0dd431bab9309f69be490f9998b4ddb93b641b09ba1f613cf",
|
||
|
|
"mmlu_astronomy": "3e8d7c33bd0431518a3d5d4b1666bdabccf46c4cc4cfe95a3886cb2266743f86",
|
||
|
|
"mmlu_business_ethics": "6ec93578c18c7bba58a384ecaa7e6e84c5e681262b34b1caa76abbbb0f9ebb02",
|
||
|
|
"mmlu_clinical_knowledge": "cf39fec90aa76a379e440c1f0d5f5b04d9f32edc4004ffcfb8eadcc25c18b118",
|
||
|
|
"mmlu_college_biology": "5378a297efa1cd71344228c95dcb29ba54757c7e527261400a4d65fd8ceed92b",
|
||
|
|
"mmlu_college_chemistry": "714e60ea7e340f0344ffde41b247a19b68e6384152a1bdc9acc0ecf24cd0e7cd",
|
||
|
|
"mmlu_college_computer_science": "2b66d244f98e61e6a78e5ed48b728e0ca2ca81c7757ccdd086fe6b0c1cf02da1",
|
||
|
|
"mmlu_college_mathematics": "90b1ce25d7d38abd925181ae3aad095cfd783375fe8de0607a57420f47454f0f",
|
||
|
|
"mmlu_college_medicine": "00cc41d26670913bdee6181e5b602726dde9f03e6a43236c2bf57486149845be",
|
||
|
|
"mmlu_college_physics": "2abddb12a247904f9ba60b39766b19a8967c24fe8eee021022a353b93d73742c",
|
||
|
|
"mmlu_computer_security": "8ffba21cebb9e39d5794c848a22e7e37cddff5c59f7b47a1d89872d9e8704b0c",
|
||
|
|
"mmlu_conceptual_physics": "c0da3219fe7e5292dad2aa7e72790e3eba951711007cc57618b0850d941b85ec",
|
||
|
|
"mmlu_econometrics": "bc667156b739500e02f7c8cb8f9621b6d62ea2011e1eaa3404cb45f6d16778ec",
|
||
|
|
"mmlu_electrical_engineering": "0a02cb16a31ec7debba4190ab1cc31afadb9416764e0b074b9b96553f346b892",
|
||
|
|
"mmlu_elementary_mathematics": "6048f1506468e757010d537a0ceedaadc587c18159a03e7664b5994843afedc2",
|
||
|
|
"mmlu_formal_logic": "5fef452fe37d3c5aefce6276a26a839e5ee5d655d72adf50bbf0a8ae0e8d74e4",
|
||
|
|
"mmlu_global_facts": "6163d6678b902c8cb5cd6c50f59bc0f31a9a356e977fcd6184614f48e3320e7a",
|
||
|
|
"mmlu_high_school_biology": "2aabe83bb100a352d24a0136c24e612b68490bbe818c33f06cc6cee5379ecad8",
|
||
|
|
"mmlu_high_school_chemistry": "dd3321fd42c8717094f57dd0eaeab0002d9ed192518a7de7aaef4913796c30f0",
|
||
|
|
"mmlu_high_school_computer_science": "888aabb9a7ac77299e8b2b8429fa44c1fbbb5ac8165f556a0b00b347e51de3b5",
|
||
|
|
"mmlu_high_school_european_history": "0b7be1963c7e224bffc18d72fe277408c4980f854a5fc94805fc008133640f44",
|
||
|
|
"mmlu_high_school_geography": "915db5f9f9c584c6886ae92cfc28ad9c364bf685f22c15009ad8bf064b4a8027",
|
||
|
|
"mmlu_high_school_government_and_politics": "411c6c6c179d1e2a252fab88f33123721540fc70f8223310e1f1349ec0b67753",
|
||
|
|
"mmlu_high_school_macroeconomics": "291b29471838255d1a2dd704960610e98b7e3f2fd001e3aabc6b25be0c236e29",
|
||
|
|
"mmlu_high_school_mathematics": "9b9567fc5c7875065ac91eac69621904f201cab822621440a2442a390f62b497",
|
||
|
|
"mmlu_high_school_microeconomics": "6f4a887cb73beee5414248002f75216c16d7c8ff739b37a2607c6bcfad16712b",
|
||
|
|
"mmlu_high_school_physics": "53edb93aefb6fc54fba283278bae10a58a7902902ff55edc30cb9c81745dbaa8",
|
||
|
|
"mmlu_high_school_psychology": "ce01f01ff9a3bdc3e956ada7ec7c02f13771c386d076323056ac00eec0095785",
|
||
|
|
"mmlu_high_school_statistics": "e0d652bf0cb6f927b0fc738e58a141f0ba8f9073615f4ddd70bf7a3b97998f21",
|
||
|
|
"mmlu_high_school_us_history": "1b31fdc7e0d51803fa6107a0c7aa3b6968031bc449ed3513a7349956fa0b53d3",
|
||
|
|
"mmlu_high_school_world_history": "9e352460adb245947aaf9671f135ae6484e44a115312955c45c7caaa0688f160",
|
||
|
|
"mmlu_human_aging": "4b4276095e96dc6e2872517b2becc2ebe9fad0f42b321486f6f2ceaef81f385e",
|
||
|
|
"mmlu_human_sexuality": "9395014a475c63632383587439df5c48c5dd3333eaa8d10ef9e0eaabd0997c3a",
|
||
|
|
"mmlu_international_law": "3d2036ffe61014fa8a2de679c6bee2c0c3ade112f03cd55e4a6fba5bdc56bf7b",
|
||
|
|
"mmlu_jurisprudence": "1548c06ef17823048b473d028f4889b3cf014beac4ac808521898ee141a34d23",
|
||
|
|
"mmlu_logical_fallacies": "66f9691122f3fa813b86fda1131037c03cf42babe650c7374924f41882716332",
|
||
|
|
"mmlu_machine_learning": "13ad1d3650a26d655a2a2eaa3be1f106d990e5b693b0e9006f8ebef221d2ed82",
|
||
|
|
"mmlu_management": "84a18a4a3039f23b7dd18838c89a80171025a1a45eda3305464a3cfcd8257b69",
|
||
|
|
"mmlu_marketing": "d6382d405e202221a78087356861723acb445fbd6f1db94152201f788b5c0606",
|
||
|
|
"mmlu_medical_genetics": "b4546cc241c5811dddfec47d0bba37122fbd65545c93d60f50b993e810d37e2d",
|
||
|
|
"mmlu_miscellaneous": "ecf4c3e0dc895ca5b7803d1b3f01fe3027b461f900ac001d36f5dd750c8eca1d",
|
||
|
|
"mmlu_moral_disputes": "84f580166c00063790619588b4f9fcd2f800a6a04778f08c1bada043bfdc6ec0",
|
||
|
|
"mmlu_moral_scenarios": "0dda4f43fd25e66311e64f4590698bfcc4c00c24ac58dbfec74b613512280ffc",
|
||
|
|
"mmlu_nutrition": "732959f39ac0c6c0e9c3823082d3669467531cebfb5f7550d79bcc31e5cfdbce",
|
||
|
|
"mmlu_philosophy": "ff6284cd5c6bfdb17898ef97dabc97916b70fc7485c57543edabdd44b1d8b543",
|
||
|
|
"mmlu_prehistory": "efb58ea420f5b7975ff34766ddc7470676e1ababa0b4705bfdf150f6871fdd83",
|
||
|
|
"mmlu_professional_accounting": "3c555bc35f86d097f509755f487781d2bd288de5dfe9aa59d8009a1dd0583c45",
|
||
|
|
"mmlu_professional_law": "b108c04f61cbbb4b9351de3730d7e3396710c7702c6cf75703155dbfdbc7819d",
|
||
|
|
"mmlu_professional_medicine": "0821fa48ca3715604bbef122215aea4e55f218dad6ba5191da2eb831971ac6f9",
|
||
|
|
"mmlu_professional_psychology": "ab273adb58e2b365b3811e4ed4c6485c9630f78058580273703eb2573dde9010",
|
||
|
|
"mmlu_public_relations": "da7dfa166f545d33a61c930908eee4f8d8544cad6328569b1c4b341becaf68f1",
|
||
|
|
"mmlu_security_studies": "85ea8f6c3b791998f3c1d57164b6e28bcf4498f65fc1988dfa4a6a65d31fb505",
|
||
|
|
"mmlu_sociology": "055d024e9587edc811c54e6cdd8fd74d535c6f5b1aad17cfd5f1598a1371606a",
|
||
|
|
"mmlu_us_foreign_policy": "a99e6492b6831a5df8c7f4c76edf15b500e59ab4c87dc73fdbd61d43bc704682",
|
||
|
|
"mmlu_virology": "ba4fc492ed4e9af97e30850257d2fdc5e7b53b37e7b4d68d443f7bd157aac21f",
|
||
|
|
"mmlu_world_religions": "9634e441f5f53329b4b6c760b08b1a130f8f03b3cc1902b23d07f57db76b7263"
|
||
|
|
},
|
||
|
|
"tokenizer_bos_token": [
|
||
|
|
null,
|
||
|
|
"None"
|
||
|
|
],
|
||
|
|
"tokenizer_eos_token": [
|
||
|
|
"<|im_end|>",
|
||
|
|
"151645"
|
||
|
|
],
|
||
|
|
"tokenizer_pad_token": [
|
||
|
|
"<|endoftext|>",
|
||
|
|
"151643"
|
||
|
|
],
|
||
|
|
"total_evaluation_time_seconds": "617.2125840899535",
|
||
|
|
"transformers_version": "5.11.0",
|
||
|
|
"upper_git_hash": null,
|
||
|
|
"versions": {
|
||
|
|
"mmlu": "2",
|
||
|
|
"mmlu_abstract_algebra": 1.0,
|
||
|
|
"mmlu_anatomy": 1.0,
|
||
|
|
"mmlu_astronomy": 1.0,
|
||
|
|
"mmlu_business_ethics": 1.0,
|
||
|
|
"mmlu_clinical_knowledge": 1.0,
|
||
|
|
"mmlu_college_biology": 1.0,
|
||
|
|
"mmlu_college_chemistry": 1.0,
|
||
|
|
"mmlu_college_computer_science": 1.0,
|
||
|
|
"mmlu_college_mathematics": 1.0,
|
||
|
|
"mmlu_college_medicine": 1.0,
|
||
|
|
"mmlu_college_physics": 1.0,
|
||
|
|
"mmlu_computer_security": 1.0,
|
||
|
|
"mmlu_conceptual_physics": 1.0,
|
||
|
|
"mmlu_econometrics": 1.0,
|
||
|
|
"mmlu_electrical_engineering": 1.0,
|
||
|
|
"mmlu_elementary_mathematics": 1.0,
|
||
|
|
"mmlu_formal_logic": 1.0,
|
||
|
|
"mmlu_global_facts": 1.0,
|
||
|
|
"mmlu_high_school_biology": 1.0,
|
||
|
|
"mmlu_high_school_chemistry": 1.0,
|
||
|
|
"mmlu_high_school_computer_science": 1.0,
|
||
|
|
"mmlu_high_school_european_history": 1.0,
|
||
|
|
"mmlu_high_school_geography": 1.0,
|
||
|
|
"mmlu_high_school_government_and_politics": 1.0,
|
||
|
|
"mmlu_high_school_macroeconomics": 1.0,
|
||
|
|
"mmlu_high_school_mathematics": 1.0,
|
||
|
|
"mmlu_high_school_microeconomics": 1.0,
|
||
|
|
"mmlu_high_school_physics": 1.0,
|
||
|
|
"mmlu_high_school_psychology": 1.0,
|
||
|
|
"mmlu_high_school_statistics": 1.0,
|
||
|
|
"mmlu_high_school_us_history": 1.0,
|
||
|
|
"mmlu_high_school_world_history": 1.0,
|
||
|
|
"mmlu_human_aging": 1.0,
|
||
|
|
"mmlu_human_sexuality": 1.0,
|
||
|
|
"mmlu_humanities": "2",
|
||
|
|
"mmlu_international_law": 1.0,
|
||
|
|
"mmlu_jurisprudence": 1.0,
|
||
|
|
"mmlu_logical_fallacies": 1.0,
|
||
|
|
"mmlu_machine_learning": 1.0,
|
||
|
|
"mmlu_management": 1.0,
|
||
|
|
"mmlu_marketing": 1.0,
|
||
|
|
"mmlu_medical_genetics": 1.0,
|
||
|
|
"mmlu_miscellaneous": 1.0,
|
||
|
|
"mmlu_moral_disputes": 1.0,
|
||
|
|
"mmlu_moral_scenarios": 1.0,
|
||
|
|
"mmlu_nutrition": 1.0,
|
||
|
|
"mmlu_other": "2",
|
||
|
|
"mmlu_philosophy": 1.0,
|
||
|
|
"mmlu_prehistory": 1.0,
|
||
|
|
"mmlu_professional_accounting": 1.0,
|
||
|
|
"mmlu_professional_law": 1.0,
|
||
|
|
"mmlu_professional_medicine": 1.0,
|
||
|
|
"mmlu_professional_psychology": 1.0,
|
||
|
|
"mmlu_public_relations": 1.0,
|
||
|
|
"mmlu_security_studies": 1.0,
|
||
|
|
"mmlu_social_sciences": "2",
|
||
|
|
"mmlu_sociology": 1.0,
|
||
|
|
"mmlu_stem": "2",
|
||
|
|
"mmlu_us_foreign_policy": 1.0,
|
||
|
|
"mmlu_virology": 1.0,
|
||
|
|
"mmlu_world_religions": 1.0
|
||
|
|
}
|
||
|
|
}
|