{
"chat_template": "{%- if tools %}\n {{- '<|im_start|>system\\n' }}\n {%- if messages[0].role == 'system' %}\n {{- messages[0].content + '\\n\\n' }}\n {%- endif %}\n {{- \"# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within XML tags:\\n\" }}\n {%- for tool in tools %}\n {{- \"\\n\" }}\n {{- tool | tojson }}\n {%- endfor %}\n {{- \"\\n\\n\\nFor each function call, return a json object with function name and arguments within XML tags:\\n\\n{\\\"name\\\": , \\\"arguments\\\": }\\n<|im_end|>\\n\" }}\n{%- else %}\n {%- if messages[0].role == 'system' %}\n {{- '<|im_start|>system\\n' + messages[0].content + '<|im_end|>\\n' }}\n {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n {%- if message.content is string %}\n {%- set content = message.content %}\n {%- else %}\n {%- set content = '' %}\n {%- endif %}\n {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) %}\n {{- '<|im_start|>' + message.role + '\\n' + content + '<|im_end|>' + '\\n' }}\n {%- elif message.role == \"assistant\" %}\n {{- '<|im_start|>' + message.role + '\\n' + content }}\n {%- if message.tool_calls %}\n {%- for tool_call in message.tool_calls %}\n {%- if (loop.first and content) or (not loop.first) %}\n {{- '\\n' }}\n {%- endif %}\n {%- if tool_call.function %}\n {%- set tool_call = tool_call.function %}\n {%- endif %}\n {{- '\\n{\"name\": \"' }}\n {{- tool_call.name }}\n {{- '\", \"arguments\": ' }}\n {%- if tool_call.arguments is string %}\n {{- tool_call.arguments }}\n {%- else %}\n {{- tool_call.arguments | tojson }}\n {%- endif %}\n {{- '}\\n' }}\n {%- endfor %}\n {%- endif %}\n {{- '<|im_end|>\\n' }}\n {%- elif message.role == \"tool\" %}\n {%- if loop.first or (messages[loop.index0 - 1].role != \"tool\") %}\n {{- '<|im_start|>user' }}\n {%- endif %}\n {{- '\\n\\n' }}\n {{- content }}\n {{- '\\n' }}\n {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<|im_start|>assistant\\n' }}\n{%- endif %}",
"chat_template_sha": "64f85b198065d0fba2a81f37e10ed68161ce2c19a754c7100e67e0ca2ee9c326",
"config": {
"batch_size": "32",
"batch_sizes": [],
"bootstrap_iters": 100000,
"device": "cuda:0",
"fewshot_seed": 1234,
"gen_kwargs": {},
"limit": null,
"model": "vllm",
"model_args": "pretrained=kosa-labs/kosa-4B-it-v1",
"numpy_seed": 1234,
"random_seed": 0,
"torch_seed": 1234,
"use_cache": "[redacted]"
},
"configs": {
"mmlu_abstract_algebra": {
"dataset_name": "abstract_algebra",
"dataset_path": "cais/mmlu",
"description": "The following are multiple choice questions (with answers) about abstract algebra.\n\n",
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_config": {
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_delimiter": "\n\n",
"fewshot_indices": null,
"gen_prefix": null,
"process_docs": null,
"sampler": "first_n",
"samples": null,
"split": "dev",
"target_delimiter": " "
},
"fewshot_delimiter": "\n\n",
"fewshot_split": "dev",
"metadata": {
"config_source": "[redacted]",
"dtype": "bfloat16",
"gpu_memory_utilization": 0.82,
"pretrained": "[redacted]",
"tensor_parallel_size": 1,
"trust_remote_code": true,
"version": 1.0
},
"metric_list": [
{
"aggregation": "mean",
"higher_is_better": true,
"metric": "acc"
}
],
"num_fewshot": 0,
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"target_delimiter": " ",
"task": "mmlu_abstract_algebra",
"task_alias": "abstract_algebra",
"test_split": "test",
"unsafe_code": false
},
"mmlu_anatomy": {
"dataset_name": "anatomy",
"dataset_path": "cais/mmlu",
"description": "The following are multiple choice questions (with answers) about anatomy.\n\n",
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_config": {
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_delimiter": "\n\n",
"fewshot_indices": null,
"gen_prefix": null,
"process_docs": null,
"sampler": "first_n",
"samples": null,
"split": "dev",
"target_delimiter": " "
},
"fewshot_delimiter": "\n\n",
"fewshot_split": "dev",
"metadata": {
"config_source": "[redacted]",
"dtype": "bfloat16",
"gpu_memory_utilization": 0.82,
"pretrained": "[redacted]",
"tensor_parallel_size": 1,
"trust_remote_code": true,
"version": 1.0
},
"metric_list": [
{
"aggregation": "mean",
"higher_is_better": true,
"metric": "acc"
}
],
"num_fewshot": 0,
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"target_delimiter": " ",
"task": "mmlu_anatomy",
"task_alias": "anatomy",
"test_split": "test",
"unsafe_code": false
},
"mmlu_astronomy": {
"dataset_name": "astronomy",
"dataset_path": "cais/mmlu",
"description": "The following are multiple choice questions (with answers) about astronomy.\n\n",
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_config": {
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_delimiter": "\n\n",
"fewshot_indices": null,
"gen_prefix": null,
"process_docs": null,
"sampler": "first_n",
"samples": null,
"split": "dev",
"target_delimiter": " "
},
"fewshot_delimiter": "\n\n",
"fewshot_split": "dev",
"metadata": {
"config_source": "[redacted]",
"dtype": "bfloat16",
"gpu_memory_utilization": 0.82,
"pretrained": "[redacted]",
"tensor_parallel_size": 1,
"trust_remote_code": true,
"version": 1.0
},
"metric_list": [
{
"aggregation": "mean",
"higher_is_better": true,
"metric": "acc"
}
],
"num_fewshot": 0,
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"target_delimiter": " ",
"task": "mmlu_astronomy",
"task_alias": "astronomy",
"test_split": "test",
"unsafe_code": false
},
"mmlu_business_ethics": {
"dataset_name": "business_ethics",
"dataset_path": "cais/mmlu",
"description": "The following are multiple choice questions (with answers) about business ethics.\n\n",
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_config": {
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_delimiter": "\n\n",
"fewshot_indices": null,
"gen_prefix": null,
"process_docs": null,
"sampler": "first_n",
"samples": null,
"split": "dev",
"target_delimiter": " "
},
"fewshot_delimiter": "\n\n",
"fewshot_split": "dev",
"metadata": {
"config_source": "[redacted]",
"dtype": "bfloat16",
"gpu_memory_utilization": 0.82,
"pretrained": "[redacted]",
"tensor_parallel_size": 1,
"trust_remote_code": true,
"version": 1.0
},
"metric_list": [
{
"aggregation": "mean",
"higher_is_better": true,
"metric": "acc"
}
],
"num_fewshot": 0,
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"target_delimiter": " ",
"task": "mmlu_business_ethics",
"task_alias": "business_ethics",
"test_split": "test",
"unsafe_code": false
},
"mmlu_clinical_knowledge": {
"dataset_name": "clinical_knowledge",
"dataset_path": "cais/mmlu",
"description": "The following are multiple choice questions (with answers) about clinical knowledge.\n\n",
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_config": {
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_delimiter": "\n\n",
"fewshot_indices": null,
"gen_prefix": null,
"process_docs": null,
"sampler": "first_n",
"samples": null,
"split": "dev",
"target_delimiter": " "
},
"fewshot_delimiter": "\n\n",
"fewshot_split": "dev",
"metadata": {
"config_source": "[redacted]",
"dtype": "bfloat16",
"gpu_memory_utilization": 0.82,
"pretrained": "[redacted]",
"tensor_parallel_size": 1,
"trust_remote_code": true,
"version": 1.0
},
"metric_list": [
{
"aggregation": "mean",
"higher_is_better": true,
"metric": "acc"
}
],
"num_fewshot": 0,
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"target_delimiter": " ",
"task": "mmlu_clinical_knowledge",
"task_alias": "clinical_knowledge",
"test_split": "test",
"unsafe_code": false
},
"mmlu_college_biology": {
"dataset_name": "college_biology",
"dataset_path": "cais/mmlu",
"description": "The following are multiple choice questions (with answers) about college biology.\n\n",
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_config": {
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_delimiter": "\n\n",
"fewshot_indices": null,
"gen_prefix": null,
"process_docs": null,
"sampler": "first_n",
"samples": null,
"split": "dev",
"target_delimiter": " "
},
"fewshot_delimiter": "\n\n",
"fewshot_split": "dev",
"metadata": {
"config_source": "[redacted]",
"dtype": "bfloat16",
"gpu_memory_utilization": 0.82,
"pretrained": "[redacted]",
"tensor_parallel_size": 1,
"trust_remote_code": true,
"version": 1.0
},
"metric_list": [
{
"aggregation": "mean",
"higher_is_better": true,
"metric": "acc"
}
],
"num_fewshot": 0,
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"target_delimiter": " ",
"task": "mmlu_college_biology",
"task_alias": "college_biology",
"test_split": "test",
"unsafe_code": false
},
"mmlu_college_chemistry": {
"dataset_name": "college_chemistry",
"dataset_path": "cais/mmlu",
"description": "The following are multiple choice questions (with answers) about college chemistry.\n\n",
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_config": {
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_delimiter": "\n\n",
"fewshot_indices": null,
"gen_prefix": null,
"process_docs": null,
"sampler": "first_n",
"samples": null,
"split": "dev",
"target_delimiter": " "
},
"fewshot_delimiter": "\n\n",
"fewshot_split": "dev",
"metadata": {
"config_source": "[redacted]",
"dtype": "bfloat16",
"gpu_memory_utilization": 0.82,
"pretrained": "[redacted]",
"tensor_parallel_size": 1,
"trust_remote_code": true,
"version": 1.0
},
"metric_list": [
{
"aggregation": "mean",
"higher_is_better": true,
"metric": "acc"
}
],
"num_fewshot": 0,
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"target_delimiter": " ",
"task": "mmlu_college_chemistry",
"task_alias": "college_chemistry",
"test_split": "test",
"unsafe_code": false
},
"mmlu_college_computer_science": {
"dataset_name": "college_computer_science",
"dataset_path": "cais/mmlu",
"description": "The following are multiple choice questions (with answers) about college computer science.\n\n",
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_config": {
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_delimiter": "\n\n",
"fewshot_indices": null,
"gen_prefix": null,
"process_docs": null,
"sampler": "first_n",
"samples": null,
"split": "dev",
"target_delimiter": " "
},
"fewshot_delimiter": "\n\n",
"fewshot_split": "dev",
"metadata": {
"config_source": "[redacted]",
"dtype": "bfloat16",
"gpu_memory_utilization": 0.82,
"pretrained": "[redacted]",
"tensor_parallel_size": 1,
"trust_remote_code": true,
"version": 1.0
},
"metric_list": [
{
"aggregation": "mean",
"higher_is_better": true,
"metric": "acc"
}
],
"num_fewshot": 0,
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"target_delimiter": " ",
"task": "mmlu_college_computer_science",
"task_alias": "college_computer_science",
"test_split": "test",
"unsafe_code": false
},
"mmlu_college_mathematics": {
"dataset_name": "college_mathematics",
"dataset_path": "cais/mmlu",
"description": "The following are multiple choice questions (with answers) about college mathematics.\n\n",
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_config": {
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_delimiter": "\n\n",
"fewshot_indices": null,
"gen_prefix": null,
"process_docs": null,
"sampler": "first_n",
"samples": null,
"split": "dev",
"target_delimiter": " "
},
"fewshot_delimiter": "\n\n",
"fewshot_split": "dev",
"metadata": {
"config_source": "[redacted]",
"dtype": "bfloat16",
"gpu_memory_utilization": 0.82,
"pretrained": "[redacted]",
"tensor_parallel_size": 1,
"trust_remote_code": true,
"version": 1.0
},
"metric_list": [
{
"aggregation": "mean",
"higher_is_better": true,
"metric": "acc"
}
],
"num_fewshot": 0,
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"target_delimiter": " ",
"task": "mmlu_college_mathematics",
"task_alias": "college_mathematics",
"test_split": "test",
"unsafe_code": false
},
"mmlu_college_medicine": {
"dataset_name": "college_medicine",
"dataset_path": "cais/mmlu",
"description": "The following are multiple choice questions (with answers) about college medicine.\n\n",
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_config": {
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_delimiter": "\n\n",
"fewshot_indices": null,
"gen_prefix": null,
"process_docs": null,
"sampler": "first_n",
"samples": null,
"split": "dev",
"target_delimiter": " "
},
"fewshot_delimiter": "\n\n",
"fewshot_split": "dev",
"metadata": {
"config_source": "[redacted]",
"dtype": "bfloat16",
"gpu_memory_utilization": 0.82,
"pretrained": "[redacted]",
"tensor_parallel_size": 1,
"trust_remote_code": true,
"version": 1.0
},
"metric_list": [
{
"aggregation": "mean",
"higher_is_better": true,
"metric": "acc"
}
],
"num_fewshot": 0,
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"target_delimiter": " ",
"task": "mmlu_college_medicine",
"task_alias": "college_medicine",
"test_split": "test",
"unsafe_code": false
},
"mmlu_college_physics": {
"dataset_name": "college_physics",
"dataset_path": "cais/mmlu",
"description": "The following are multiple choice questions (with answers) about college physics.\n\n",
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_config": {
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_delimiter": "\n\n",
"fewshot_indices": null,
"gen_prefix": null,
"process_docs": null,
"sampler": "first_n",
"samples": null,
"split": "dev",
"target_delimiter": " "
},
"fewshot_delimiter": "\n\n",
"fewshot_split": "dev",
"metadata": {
"config_source": "[redacted]",
"dtype": "bfloat16",
"gpu_memory_utilization": 0.82,
"pretrained": "[redacted]",
"tensor_parallel_size": 1,
"trust_remote_code": true,
"version": 1.0
},
"metric_list": [
{
"aggregation": "mean",
"higher_is_better": true,
"metric": "acc"
}
],
"num_fewshot": 0,
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"target_delimiter": " ",
"task": "mmlu_college_physics",
"task_alias": "college_physics",
"test_split": "test",
"unsafe_code": false
},
"mmlu_computer_security": {
"dataset_name": "computer_security",
"dataset_path": "cais/mmlu",
"description": "The following are multiple choice questions (with answers) about computer security.\n\n",
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_config": {
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_delimiter": "\n\n",
"fewshot_indices": null,
"gen_prefix": null,
"process_docs": null,
"sampler": "first_n",
"samples": null,
"split": "dev",
"target_delimiter": " "
},
"fewshot_delimiter": "\n\n",
"fewshot_split": "dev",
"metadata": {
"config_source": "[redacted]",
"dtype": "bfloat16",
"gpu_memory_utilization": 0.82,
"pretrained": "[redacted]",
"tensor_parallel_size": 1,
"trust_remote_code": true,
"version": 1.0
},
"metric_list": [
{
"aggregation": "mean",
"higher_is_better": true,
"metric": "acc"
}
],
"num_fewshot": 0,
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"target_delimiter": " ",
"task": "mmlu_computer_security",
"task_alias": "computer_security",
"test_split": "test",
"unsafe_code": false
},
"mmlu_conceptual_physics": {
"dataset_name": "conceptual_physics",
"dataset_path": "cais/mmlu",
"description": "The following are multiple choice questions (with answers) about conceptual physics.\n\n",
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_config": {
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_delimiter": "\n\n",
"fewshot_indices": null,
"gen_prefix": null,
"process_docs": null,
"sampler": "first_n",
"samples": null,
"split": "dev",
"target_delimiter": " "
},
"fewshot_delimiter": "\n\n",
"fewshot_split": "dev",
"metadata": {
"config_source": "[redacted]",
"dtype": "bfloat16",
"gpu_memory_utilization": 0.82,
"pretrained": "[redacted]",
"tensor_parallel_size": 1,
"trust_remote_code": true,
"version": 1.0
},
"metric_list": [
{
"aggregation": "mean",
"higher_is_better": true,
"metric": "acc"
}
],
"num_fewshot": 0,
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"target_delimiter": " ",
"task": "mmlu_conceptual_physics",
"task_alias": "conceptual_physics",
"test_split": "test",
"unsafe_code": false
},
"mmlu_econometrics": {
"dataset_name": "econometrics",
"dataset_path": "cais/mmlu",
"description": "The following are multiple choice questions (with answers) about econometrics.\n\n",
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_config": {
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_delimiter": "\n\n",
"fewshot_indices": null,
"gen_prefix": null,
"process_docs": null,
"sampler": "first_n",
"samples": null,
"split": "dev",
"target_delimiter": " "
},
"fewshot_delimiter": "\n\n",
"fewshot_split": "dev",
"metadata": {
"config_source": "[redacted]",
"dtype": "bfloat16",
"gpu_memory_utilization": 0.82,
"pretrained": "[redacted]",
"tensor_parallel_size": 1,
"trust_remote_code": true,
"version": 1.0
},
"metric_list": [
{
"aggregation": "mean",
"higher_is_better": true,
"metric": "acc"
}
],
"num_fewshot": 0,
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"target_delimiter": " ",
"task": "mmlu_econometrics",
"task_alias": "econometrics",
"test_split": "test",
"unsafe_code": false
},
"mmlu_electrical_engineering": {
"dataset_name": "electrical_engineering",
"dataset_path": "cais/mmlu",
"description": "The following are multiple choice questions (with answers) about electrical engineering.\n\n",
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_config": {
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_delimiter": "\n\n",
"fewshot_indices": null,
"gen_prefix": null,
"process_docs": null,
"sampler": "first_n",
"samples": null,
"split": "dev",
"target_delimiter": " "
},
"fewshot_delimiter": "\n\n",
"fewshot_split": "dev",
"metadata": {
"config_source": "[redacted]",
"dtype": "bfloat16",
"gpu_memory_utilization": 0.82,
"pretrained": "[redacted]",
"tensor_parallel_size": 1,
"trust_remote_code": true,
"version": 1.0
},
"metric_list": [
{
"aggregation": "mean",
"higher_is_better": true,
"metric": "acc"
}
],
"num_fewshot": 0,
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"target_delimiter": " ",
"task": "mmlu_electrical_engineering",
"task_alias": "electrical_engineering",
"test_split": "test",
"unsafe_code": false
},
"mmlu_elementary_mathematics": {
"dataset_name": "elementary_mathematics",
"dataset_path": "cais/mmlu",
"description": "The following are multiple choice questions (with answers) about elementary mathematics.\n\n",
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_config": {
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_delimiter": "\n\n",
"fewshot_indices": null,
"gen_prefix": null,
"process_docs": null,
"sampler": "first_n",
"samples": null,
"split": "dev",
"target_delimiter": " "
},
"fewshot_delimiter": "\n\n",
"fewshot_split": "dev",
"metadata": {
"config_source": "[redacted]",
"dtype": "bfloat16",
"gpu_memory_utilization": 0.82,
"pretrained": "[redacted]",
"tensor_parallel_size": 1,
"trust_remote_code": true,
"version": 1.0
},
"metric_list": [
{
"aggregation": "mean",
"higher_is_better": true,
"metric": "acc"
}
],
"num_fewshot": 0,
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"target_delimiter": " ",
"task": "mmlu_elementary_mathematics",
"task_alias": "elementary_mathematics",
"test_split": "test",
"unsafe_code": false
},
"mmlu_formal_logic": {
"dataset_name": "formal_logic",
"dataset_path": "cais/mmlu",
"description": "The following are multiple choice questions (with answers) about formal logic.\n\n",
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_config": {
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_delimiter": "\n\n",
"fewshot_indices": null,
"gen_prefix": null,
"process_docs": null,
"sampler": "first_n",
"samples": null,
"split": "dev",
"target_delimiter": " "
},
"fewshot_delimiter": "\n\n",
"fewshot_split": "dev",
"metadata": {
"config_source": "[redacted]",
"dtype": "bfloat16",
"gpu_memory_utilization": 0.82,
"pretrained": "[redacted]",
"tensor_parallel_size": 1,
"trust_remote_code": true,
"version": 1.0
},
"metric_list": [
{
"aggregation": "mean",
"higher_is_better": true,
"metric": "acc"
}
],
"num_fewshot": 0,
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"target_delimiter": " ",
"task": "mmlu_formal_logic",
"task_alias": "formal_logic",
"test_split": "test",
"unsafe_code": false
},
"mmlu_global_facts": {
"dataset_name": "global_facts",
"dataset_path": "cais/mmlu",
"description": "The following are multiple choice questions (with answers) about global facts.\n\n",
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_config": {
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_delimiter": "\n\n",
"fewshot_indices": null,
"gen_prefix": null,
"process_docs": null,
"sampler": "first_n",
"samples": null,
"split": "dev",
"target_delimiter": " "
},
"fewshot_delimiter": "\n\n",
"fewshot_split": "dev",
"metadata": {
"config_source": "[redacted]",
"dtype": "bfloat16",
"gpu_memory_utilization": 0.82,
"pretrained": "[redacted]",
"tensor_parallel_size": 1,
"trust_remote_code": true,
"version": 1.0
},
"metric_list": [
{
"aggregation": "mean",
"higher_is_better": true,
"metric": "acc"
}
],
"num_fewshot": 0,
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"target_delimiter": " ",
"task": "mmlu_global_facts",
"task_alias": "global_facts",
"test_split": "test",
"unsafe_code": false
},
"mmlu_high_school_biology": {
"dataset_name": "high_school_biology",
"dataset_path": "cais/mmlu",
"description": "The following are multiple choice questions (with answers) about high school biology.\n\n",
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_config": {
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_delimiter": "\n\n",
"fewshot_indices": null,
"gen_prefix": null,
"process_docs": null,
"sampler": "first_n",
"samples": null,
"split": "dev",
"target_delimiter": " "
},
"fewshot_delimiter": "\n\n",
"fewshot_split": "dev",
"metadata": {
"config_source": "[redacted]",
"dtype": "bfloat16",
"gpu_memory_utilization": 0.82,
"pretrained": "[redacted]",
"tensor_parallel_size": 1,
"trust_remote_code": true,
"version": 1.0
},
"metric_list": [
{
"aggregation": "mean",
"higher_is_better": true,
"metric": "acc"
}
],
"num_fewshot": 0,
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"target_delimiter": " ",
"task": "mmlu_high_school_biology",
"task_alias": "high_school_biology",
"test_split": "test",
"unsafe_code": false
},
"mmlu_high_school_chemistry": {
"dataset_name": "high_school_chemistry",
"dataset_path": "cais/mmlu",
"description": "The following are multiple choice questions (with answers) about high school chemistry.\n\n",
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_config": {
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_delimiter": "\n\n",
"fewshot_indices": null,
"gen_prefix": null,
"process_docs": null,
"sampler": "first_n",
"samples": null,
"split": "dev",
"target_delimiter": " "
},
"fewshot_delimiter": "\n\n",
"fewshot_split": "dev",
"metadata": {
"config_source": "[redacted]",
"dtype": "bfloat16",
"gpu_memory_utilization": 0.82,
"pretrained": "[redacted]",
"tensor_parallel_size": 1,
"trust_remote_code": true,
"version": 1.0
},
"metric_list": [
{
"aggregation": "mean",
"higher_is_better": true,
"metric": "acc"
}
],
"num_fewshot": 0,
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"target_delimiter": " ",
"task": "mmlu_high_school_chemistry",
"task_alias": "high_school_chemistry",
"test_split": "test",
"unsafe_code": false
},
"mmlu_high_school_computer_science": {
"dataset_name": "high_school_computer_science",
"dataset_path": "cais/mmlu",
"description": "The following are multiple choice questions (with answers) about high school computer science.\n\n",
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_config": {
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_delimiter": "\n\n",
"fewshot_indices": null,
"gen_prefix": null,
"process_docs": null,
"sampler": "first_n",
"samples": null,
"split": "dev",
"target_delimiter": " "
},
"fewshot_delimiter": "\n\n",
"fewshot_split": "dev",
"metadata": {
"config_source": "[redacted]",
"dtype": "bfloat16",
"gpu_memory_utilization": 0.82,
"pretrained": "[redacted]",
"tensor_parallel_size": 1,
"trust_remote_code": true,
"version": 1.0
},
"metric_list": [
{
"aggregation": "mean",
"higher_is_better": true,
"metric": "acc"
}
],
"num_fewshot": 0,
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"target_delimiter": " ",
"task": "mmlu_high_school_computer_science",
"task_alias": "high_school_computer_science",
"test_split": "test",
"unsafe_code": false
},
"mmlu_high_school_european_history": {
"dataset_name": "high_school_european_history",
"dataset_path": "cais/mmlu",
"description": "The following are multiple choice questions (with answers) about high school european history.\n\n",
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_config": {
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_delimiter": "\n\n",
"fewshot_indices": null,
"gen_prefix": null,
"process_docs": null,
"sampler": "first_n",
"samples": null,
"split": "dev",
"target_delimiter": " "
},
"fewshot_delimiter": "\n\n",
"fewshot_split": "dev",
"metadata": {
"config_source": "[redacted]",
"dtype": "bfloat16",
"gpu_memory_utilization": 0.82,
"pretrained": "[redacted]",
"tensor_parallel_size": 1,
"trust_remote_code": true,
"version": 1.0
},
"metric_list": [
{
"aggregation": "mean",
"higher_is_better": true,
"metric": "acc"
}
],
"num_fewshot": 0,
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"target_delimiter": " ",
"task": "mmlu_high_school_european_history",
"task_alias": "high_school_european_history",
"test_split": "test",
"unsafe_code": false
},
"mmlu_high_school_geography": {
"dataset_name": "high_school_geography",
"dataset_path": "cais/mmlu",
"description": "The following are multiple choice questions (with answers) about high school geography.\n\n",
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_config": {
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_delimiter": "\n\n",
"fewshot_indices": null,
"gen_prefix": null,
"process_docs": null,
"sampler": "first_n",
"samples": null,
"split": "dev",
"target_delimiter": " "
},
"fewshot_delimiter": "\n\n",
"fewshot_split": "dev",
"metadata": {
"config_source": "[redacted]",
"dtype": "bfloat16",
"gpu_memory_utilization": 0.82,
"pretrained": "[redacted]",
"tensor_parallel_size": 1,
"trust_remote_code": true,
"version": 1.0
},
"metric_list": [
{
"aggregation": "mean",
"higher_is_better": true,
"metric": "acc"
}
],
"num_fewshot": 0,
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"target_delimiter": " ",
"task": "mmlu_high_school_geography",
"task_alias": "high_school_geography",
"test_split": "test",
"unsafe_code": false
},
"mmlu_high_school_government_and_politics": {
"dataset_name": "high_school_government_and_politics",
"dataset_path": "cais/mmlu",
"description": "The following are multiple choice questions (with answers) about high school government and politics.\n\n",
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_config": {
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_delimiter": "\n\n",
"fewshot_indices": null,
"gen_prefix": null,
"process_docs": null,
"sampler": "first_n",
"samples": null,
"split": "dev",
"target_delimiter": " "
},
"fewshot_delimiter": "\n\n",
"fewshot_split": "dev",
"metadata": {
"config_source": "[redacted]",
"dtype": "bfloat16",
"gpu_memory_utilization": 0.82,
"pretrained": "[redacted]",
"tensor_parallel_size": 1,
"trust_remote_code": true,
"version": 1.0
},
"metric_list": [
{
"aggregation": "mean",
"higher_is_better": true,
"metric": "acc"
}
],
"num_fewshot": 0,
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"target_delimiter": " ",
"task": "mmlu_high_school_government_and_politics",
"task_alias": "high_school_government_and_politics",
"test_split": "test",
"unsafe_code": false
},
"mmlu_high_school_macroeconomics": {
"dataset_name": "high_school_macroeconomics",
"dataset_path": "cais/mmlu",
"description": "The following are multiple choice questions (with answers) about high school macroeconomics.\n\n",
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_config": {
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_delimiter": "\n\n",
"fewshot_indices": null,
"gen_prefix": null,
"process_docs": null,
"sampler": "first_n",
"samples": null,
"split": "dev",
"target_delimiter": " "
},
"fewshot_delimiter": "\n\n",
"fewshot_split": "dev",
"metadata": {
"config_source": "[redacted]",
"dtype": "bfloat16",
"gpu_memory_utilization": 0.82,
"pretrained": "[redacted]",
"tensor_parallel_size": 1,
"trust_remote_code": true,
"version": 1.0
},
"metric_list": [
{
"aggregation": "mean",
"higher_is_better": true,
"metric": "acc"
}
],
"num_fewshot": 0,
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"target_delimiter": " ",
"task": "mmlu_high_school_macroeconomics",
"task_alias": "high_school_macroeconomics",
"test_split": "test",
"unsafe_code": false
},
"mmlu_high_school_mathematics": {
"dataset_name": "high_school_mathematics",
"dataset_path": "cais/mmlu",
"description": "The following are multiple choice questions (with answers) about high school mathematics.\n\n",
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_config": {
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_delimiter": "\n\n",
"fewshot_indices": null,
"gen_prefix": null,
"process_docs": null,
"sampler": "first_n",
"samples": null,
"split": "dev",
"target_delimiter": " "
},
"fewshot_delimiter": "\n\n",
"fewshot_split": "dev",
"metadata": {
"config_source": "[redacted]",
"dtype": "bfloat16",
"gpu_memory_utilization": 0.82,
"pretrained": "[redacted]",
"tensor_parallel_size": 1,
"trust_remote_code": true,
"version": 1.0
},
"metric_list": [
{
"aggregation": "mean",
"higher_is_better": true,
"metric": "acc"
}
],
"num_fewshot": 0,
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"target_delimiter": " ",
"task": "mmlu_high_school_mathematics",
"task_alias": "high_school_mathematics",
"test_split": "test",
"unsafe_code": false
},
"mmlu_high_school_microeconomics": {
"dataset_name": "high_school_microeconomics",
"dataset_path": "cais/mmlu",
"description": "The following are multiple choice questions (with answers) about high school microeconomics.\n\n",
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_config": {
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_delimiter": "\n\n",
"fewshot_indices": null,
"gen_prefix": null,
"process_docs": null,
"sampler": "first_n",
"samples": null,
"split": "dev",
"target_delimiter": " "
},
"fewshot_delimiter": "\n\n",
"fewshot_split": "dev",
"metadata": {
"config_source": "[redacted]",
"dtype": "bfloat16",
"gpu_memory_utilization": 0.82,
"pretrained": "[redacted]",
"tensor_parallel_size": 1,
"trust_remote_code": true,
"version": 1.0
},
"metric_list": [
{
"aggregation": "mean",
"higher_is_better": true,
"metric": "acc"
}
],
"num_fewshot": 0,
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"target_delimiter": " ",
"task": "mmlu_high_school_microeconomics",
"task_alias": "high_school_microeconomics",
"test_split": "test",
"unsafe_code": false
},
"mmlu_high_school_physics": {
"dataset_name": "high_school_physics",
"dataset_path": "cais/mmlu",
"description": "The following are multiple choice questions (with answers) about high school physics.\n\n",
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_config": {
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_delimiter": "\n\n",
"fewshot_indices": null,
"gen_prefix": null,
"process_docs": null,
"sampler": "first_n",
"samples": null,
"split": "dev",
"target_delimiter": " "
},
"fewshot_delimiter": "\n\n",
"fewshot_split": "dev",
"metadata": {
"config_source": "[redacted]",
"dtype": "bfloat16",
"gpu_memory_utilization": 0.82,
"pretrained": "[redacted]",
"tensor_parallel_size": 1,
"trust_remote_code": true,
"version": 1.0
},
"metric_list": [
{
"aggregation": "mean",
"higher_is_better": true,
"metric": "acc"
}
],
"num_fewshot": 0,
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"target_delimiter": " ",
"task": "mmlu_high_school_physics",
"task_alias": "high_school_physics",
"test_split": "test",
"unsafe_code": false
},
"mmlu_high_school_psychology": {
"dataset_name": "high_school_psychology",
"dataset_path": "cais/mmlu",
"description": "The following are multiple choice questions (with answers) about high school psychology.\n\n",
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_config": {
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_delimiter": "\n\n",
"fewshot_indices": null,
"gen_prefix": null,
"process_docs": null,
"sampler": "first_n",
"samples": null,
"split": "dev",
"target_delimiter": " "
},
"fewshot_delimiter": "\n\n",
"fewshot_split": "dev",
"metadata": {
"config_source": "[redacted]",
"dtype": "bfloat16",
"gpu_memory_utilization": 0.82,
"pretrained": "[redacted]",
"tensor_parallel_size": 1,
"trust_remote_code": true,
"version": 1.0
},
"metric_list": [
{
"aggregation": "mean",
"higher_is_better": true,
"metric": "acc"
}
],
"num_fewshot": 0,
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"target_delimiter": " ",
"task": "mmlu_high_school_psychology",
"task_alias": "high_school_psychology",
"test_split": "test",
"unsafe_code": false
},
"mmlu_high_school_statistics": {
"dataset_name": "high_school_statistics",
"dataset_path": "cais/mmlu",
"description": "The following are multiple choice questions (with answers) about high school statistics.\n\n",
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_config": {
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_delimiter": "\n\n",
"fewshot_indices": null,
"gen_prefix": null,
"process_docs": null,
"sampler": "first_n",
"samples": null,
"split": "dev",
"target_delimiter": " "
},
"fewshot_delimiter": "\n\n",
"fewshot_split": "dev",
"metadata": {
"config_source": "[redacted]",
"dtype": "bfloat16",
"gpu_memory_utilization": 0.82,
"pretrained": "[redacted]",
"tensor_parallel_size": 1,
"trust_remote_code": true,
"version": 1.0
},
"metric_list": [
{
"aggregation": "mean",
"higher_is_better": true,
"metric": "acc"
}
],
"num_fewshot": 0,
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"target_delimiter": " ",
"task": "mmlu_high_school_statistics",
"task_alias": "high_school_statistics",
"test_split": "test",
"unsafe_code": false
},
"mmlu_high_school_us_history": {
"dataset_name": "high_school_us_history",
"dataset_path": "cais/mmlu",
"description": "The following are multiple choice questions (with answers) about high school us history.\n\n",
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_config": {
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_delimiter": "\n\n",
"fewshot_indices": null,
"gen_prefix": null,
"process_docs": null,
"sampler": "first_n",
"samples": null,
"split": "dev",
"target_delimiter": " "
},
"fewshot_delimiter": "\n\n",
"fewshot_split": "dev",
"metadata": {
"config_source": "[redacted]",
"dtype": "bfloat16",
"gpu_memory_utilization": 0.82,
"pretrained": "[redacted]",
"tensor_parallel_size": 1,
"trust_remote_code": true,
"version": 1.0
},
"metric_list": [
{
"aggregation": "mean",
"higher_is_better": true,
"metric": "acc"
}
],
"num_fewshot": 0,
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"target_delimiter": " ",
"task": "mmlu_high_school_us_history",
"task_alias": "high_school_us_history",
"test_split": "test",
"unsafe_code": false
},
"mmlu_high_school_world_history": {
"dataset_name": "high_school_world_history",
"dataset_path": "cais/mmlu",
"description": "The following are multiple choice questions (with answers) about high school world history.\n\n",
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_config": {
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_delimiter": "\n\n",
"fewshot_indices": null,
"gen_prefix": null,
"process_docs": null,
"sampler": "first_n",
"samples": null,
"split": "dev",
"target_delimiter": " "
},
"fewshot_delimiter": "\n\n",
"fewshot_split": "dev",
"metadata": {
"config_source": "[redacted]",
"dtype": "bfloat16",
"gpu_memory_utilization": 0.82,
"pretrained": "[redacted]",
"tensor_parallel_size": 1,
"trust_remote_code": true,
"version": 1.0
},
"metric_list": [
{
"aggregation": "mean",
"higher_is_better": true,
"metric": "acc"
}
],
"num_fewshot": 0,
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"target_delimiter": " ",
"task": "mmlu_high_school_world_history",
"task_alias": "high_school_world_history",
"test_split": "test",
"unsafe_code": false
},
"mmlu_human_aging": {
"dataset_name": "human_aging",
"dataset_path": "cais/mmlu",
"description": "The following are multiple choice questions (with answers) about human aging.\n\n",
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_config": {
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_delimiter": "\n\n",
"fewshot_indices": null,
"gen_prefix": null,
"process_docs": null,
"sampler": "first_n",
"samples": null,
"split": "dev",
"target_delimiter": " "
},
"fewshot_delimiter": "\n\n",
"fewshot_split": "dev",
"metadata": {
"config_source": "[redacted]",
"dtype": "bfloat16",
"gpu_memory_utilization": 0.82,
"pretrained": "[redacted]",
"tensor_parallel_size": 1,
"trust_remote_code": true,
"version": 1.0
},
"metric_list": [
{
"aggregation": "mean",
"higher_is_better": true,
"metric": "acc"
}
],
"num_fewshot": 0,
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"target_delimiter": " ",
"task": "mmlu_human_aging",
"task_alias": "human_aging",
"test_split": "test",
"unsafe_code": false
},
"mmlu_human_sexuality": {
"dataset_name": "human_sexuality",
"dataset_path": "cais/mmlu",
"description": "The following are multiple choice questions (with answers) about human sexuality.\n\n",
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_config": {
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_delimiter": "\n\n",
"fewshot_indices": null,
"gen_prefix": null,
"process_docs": null,
"sampler": "first_n",
"samples": null,
"split": "dev",
"target_delimiter": " "
},
"fewshot_delimiter": "\n\n",
"fewshot_split": "dev",
"metadata": {
"config_source": "[redacted]",
"dtype": "bfloat16",
"gpu_memory_utilization": 0.82,
"pretrained": "[redacted]",
"tensor_parallel_size": 1,
"trust_remote_code": true,
"version": 1.0
},
"metric_list": [
{
"aggregation": "mean",
"higher_is_better": true,
"metric": "acc"
}
],
"num_fewshot": 0,
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"target_delimiter": " ",
"task": "mmlu_human_sexuality",
"task_alias": "human_sexuality",
"test_split": "test",
"unsafe_code": false
},
"mmlu_international_law": {
"dataset_name": "international_law",
"dataset_path": "cais/mmlu",
"description": "The following are multiple choice questions (with answers) about international law.\n\n",
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_config": {
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_delimiter": "\n\n",
"fewshot_indices": null,
"gen_prefix": null,
"process_docs": null,
"sampler": "first_n",
"samples": null,
"split": "dev",
"target_delimiter": " "
},
"fewshot_delimiter": "\n\n",
"fewshot_split": "dev",
"metadata": {
"config_source": "[redacted]",
"dtype": "bfloat16",
"gpu_memory_utilization": 0.82,
"pretrained": "[redacted]",
"tensor_parallel_size": 1,
"trust_remote_code": true,
"version": 1.0
},
"metric_list": [
{
"aggregation": "mean",
"higher_is_better": true,
"metric": "acc"
}
],
"num_fewshot": 0,
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"target_delimiter": " ",
"task": "mmlu_international_law",
"task_alias": "international_law",
"test_split": "test",
"unsafe_code": false
},
"mmlu_jurisprudence": {
"dataset_name": "jurisprudence",
"dataset_path": "cais/mmlu",
"description": "The following are multiple choice questions (with answers) about jurisprudence.\n\n",
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_config": {
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_delimiter": "\n\n",
"fewshot_indices": null,
"gen_prefix": null,
"process_docs": null,
"sampler": "first_n",
"samples": null,
"split": "dev",
"target_delimiter": " "
},
"fewshot_delimiter": "\n\n",
"fewshot_split": "dev",
"metadata": {
"config_source": "[redacted]",
"dtype": "bfloat16",
"gpu_memory_utilization": 0.82,
"pretrained": "[redacted]",
"tensor_parallel_size": 1,
"trust_remote_code": true,
"version": 1.0
},
"metric_list": [
{
"aggregation": "mean",
"higher_is_better": true,
"metric": "acc"
}
],
"num_fewshot": 0,
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"target_delimiter": " ",
"task": "mmlu_jurisprudence",
"task_alias": "jurisprudence",
"test_split": "test",
"unsafe_code": false
},
"mmlu_logical_fallacies": {
"dataset_name": "logical_fallacies",
"dataset_path": "cais/mmlu",
"description": "The following are multiple choice questions (with answers) about logical fallacies.\n\n",
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_config": {
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_delimiter": "\n\n",
"fewshot_indices": null,
"gen_prefix": null,
"process_docs": null,
"sampler": "first_n",
"samples": null,
"split": "dev",
"target_delimiter": " "
},
"fewshot_delimiter": "\n\n",
"fewshot_split": "dev",
"metadata": {
"config_source": "[redacted]",
"dtype": "bfloat16",
"gpu_memory_utilization": 0.82,
"pretrained": "[redacted]",
"tensor_parallel_size": 1,
"trust_remote_code": true,
"version": 1.0
},
"metric_list": [
{
"aggregation": "mean",
"higher_is_better": true,
"metric": "acc"
}
],
"num_fewshot": 0,
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"target_delimiter": " ",
"task": "mmlu_logical_fallacies",
"task_alias": "logical_fallacies",
"test_split": "test",
"unsafe_code": false
},
"mmlu_machine_learning": {
"dataset_name": "machine_learning",
"dataset_path": "cais/mmlu",
"description": "The following are multiple choice questions (with answers) about machine learning.\n\n",
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_config": {
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_delimiter": "\n\n",
"fewshot_indices": null,
"gen_prefix": null,
"process_docs": null,
"sampler": "first_n",
"samples": null,
"split": "dev",
"target_delimiter": " "
},
"fewshot_delimiter": "\n\n",
"fewshot_split": "dev",
"metadata": {
"config_source": "[redacted]",
"dtype": "bfloat16",
"gpu_memory_utilization": 0.82,
"pretrained": "[redacted]",
"tensor_parallel_size": 1,
"trust_remote_code": true,
"version": 1.0
},
"metric_list": [
{
"aggregation": "mean",
"higher_is_better": true,
"metric": "acc"
}
],
"num_fewshot": 0,
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"target_delimiter": " ",
"task": "mmlu_machine_learning",
"task_alias": "machine_learning",
"test_split": "test",
"unsafe_code": false
},
"mmlu_management": {
"dataset_name": "management",
"dataset_path": "cais/mmlu",
"description": "The following are multiple choice questions (with answers) about management.\n\n",
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_config": {
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_delimiter": "\n\n",
"fewshot_indices": null,
"gen_prefix": null,
"process_docs": null,
"sampler": "first_n",
"samples": null,
"split": "dev",
"target_delimiter": " "
},
"fewshot_delimiter": "\n\n",
"fewshot_split": "dev",
"metadata": {
"config_source": "[redacted]",
"dtype": "bfloat16",
"gpu_memory_utilization": 0.82,
"pretrained": "[redacted]",
"tensor_parallel_size": 1,
"trust_remote_code": true,
"version": 1.0
},
"metric_list": [
{
"aggregation": "mean",
"higher_is_better": true,
"metric": "acc"
}
],
"num_fewshot": 0,
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"target_delimiter": " ",
"task": "mmlu_management",
"task_alias": "management",
"test_split": "test",
"unsafe_code": false
},
"mmlu_marketing": {
"dataset_name": "marketing",
"dataset_path": "cais/mmlu",
"description": "The following are multiple choice questions (with answers) about marketing.\n\n",
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_config": {
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_delimiter": "\n\n",
"fewshot_indices": null,
"gen_prefix": null,
"process_docs": null,
"sampler": "first_n",
"samples": null,
"split": "dev",
"target_delimiter": " "
},
"fewshot_delimiter": "\n\n",
"fewshot_split": "dev",
"metadata": {
"config_source": "[redacted]",
"dtype": "bfloat16",
"gpu_memory_utilization": 0.82,
"pretrained": "[redacted]",
"tensor_parallel_size": 1,
"trust_remote_code": true,
"version": 1.0
},
"metric_list": [
{
"aggregation": "mean",
"higher_is_better": true,
"metric": "acc"
}
],
"num_fewshot": 0,
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"target_delimiter": " ",
"task": "mmlu_marketing",
"task_alias": "marketing",
"test_split": "test",
"unsafe_code": false
},
"mmlu_medical_genetics": {
"dataset_name": "medical_genetics",
"dataset_path": "cais/mmlu",
"description": "The following are multiple choice questions (with answers) about medical genetics.\n\n",
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_config": {
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_delimiter": "\n\n",
"fewshot_indices": null,
"gen_prefix": null,
"process_docs": null,
"sampler": "first_n",
"samples": null,
"split": "dev",
"target_delimiter": " "
},
"fewshot_delimiter": "\n\n",
"fewshot_split": "dev",
"metadata": {
"config_source": "[redacted]",
"dtype": "bfloat16",
"gpu_memory_utilization": 0.82,
"pretrained": "[redacted]",
"tensor_parallel_size": 1,
"trust_remote_code": true,
"version": 1.0
},
"metric_list": [
{
"aggregation": "mean",
"higher_is_better": true,
"metric": "acc"
}
],
"num_fewshot": 0,
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"target_delimiter": " ",
"task": "mmlu_medical_genetics",
"task_alias": "medical_genetics",
"test_split": "test",
"unsafe_code": false
},
"mmlu_miscellaneous": {
"dataset_name": "miscellaneous",
"dataset_path": "cais/mmlu",
"description": "The following are multiple choice questions (with answers) about miscellaneous.\n\n",
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_config": {
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_delimiter": "\n\n",
"fewshot_indices": null,
"gen_prefix": null,
"process_docs": null,
"sampler": "first_n",
"samples": null,
"split": "dev",
"target_delimiter": " "
},
"fewshot_delimiter": "\n\n",
"fewshot_split": "dev",
"metadata": {
"config_source": "[redacted]",
"dtype": "bfloat16",
"gpu_memory_utilization": 0.82,
"pretrained": "[redacted]",
"tensor_parallel_size": 1,
"trust_remote_code": true,
"version": 1.0
},
"metric_list": [
{
"aggregation": "mean",
"higher_is_better": true,
"metric": "acc"
}
],
"num_fewshot": 0,
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"target_delimiter": " ",
"task": "mmlu_miscellaneous",
"task_alias": "miscellaneous",
"test_split": "test",
"unsafe_code": false
},
"mmlu_moral_disputes": {
"dataset_name": "moral_disputes",
"dataset_path": "cais/mmlu",
"description": "The following are multiple choice questions (with answers) about moral disputes.\n\n",
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_config": {
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_delimiter": "\n\n",
"fewshot_indices": null,
"gen_prefix": null,
"process_docs": null,
"sampler": "first_n",
"samples": null,
"split": "dev",
"target_delimiter": " "
},
"fewshot_delimiter": "\n\n",
"fewshot_split": "dev",
"metadata": {
"config_source": "[redacted]",
"dtype": "bfloat16",
"gpu_memory_utilization": 0.82,
"pretrained": "[redacted]",
"tensor_parallel_size": 1,
"trust_remote_code": true,
"version": 1.0
},
"metric_list": [
{
"aggregation": "mean",
"higher_is_better": true,
"metric": "acc"
}
],
"num_fewshot": 0,
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"target_delimiter": " ",
"task": "mmlu_moral_disputes",
"task_alias": "moral_disputes",
"test_split": "test",
"unsafe_code": false
},
"mmlu_moral_scenarios": {
"dataset_name": "moral_scenarios",
"dataset_path": "cais/mmlu",
"description": "The following are multiple choice questions (with answers) about moral scenarios.\n\n",
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_config": {
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_delimiter": "\n\n",
"fewshot_indices": null,
"gen_prefix": null,
"process_docs": null,
"sampler": "first_n",
"samples": null,
"split": "dev",
"target_delimiter": " "
},
"fewshot_delimiter": "\n\n",
"fewshot_split": "dev",
"metadata": {
"config_source": "[redacted]",
"dtype": "bfloat16",
"gpu_memory_utilization": 0.82,
"pretrained": "[redacted]",
"tensor_parallel_size": 1,
"trust_remote_code": true,
"version": 1.0
},
"metric_list": [
{
"aggregation": "mean",
"higher_is_better": true,
"metric": "acc"
}
],
"num_fewshot": 0,
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"target_delimiter": " ",
"task": "mmlu_moral_scenarios",
"task_alias": "moral_scenarios",
"test_split": "test",
"unsafe_code": false
},
"mmlu_nutrition": {
"dataset_name": "nutrition",
"dataset_path": "cais/mmlu",
"description": "The following are multiple choice questions (with answers) about nutrition.\n\n",
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_config": {
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_delimiter": "\n\n",
"fewshot_indices": null,
"gen_prefix": null,
"process_docs": null,
"sampler": "first_n",
"samples": null,
"split": "dev",
"target_delimiter": " "
},
"fewshot_delimiter": "\n\n",
"fewshot_split": "dev",
"metadata": {
"config_source": "[redacted]",
"dtype": "bfloat16",
"gpu_memory_utilization": 0.82,
"pretrained": "[redacted]",
"tensor_parallel_size": 1,
"trust_remote_code": true,
"version": 1.0
},
"metric_list": [
{
"aggregation": "mean",
"higher_is_better": true,
"metric": "acc"
}
],
"num_fewshot": 0,
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"target_delimiter": " ",
"task": "mmlu_nutrition",
"task_alias": "nutrition",
"test_split": "test",
"unsafe_code": false
},
"mmlu_philosophy": {
"dataset_name": "philosophy",
"dataset_path": "cais/mmlu",
"description": "The following are multiple choice questions (with answers) about philosophy.\n\n",
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_config": {
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_delimiter": "\n\n",
"fewshot_indices": null,
"gen_prefix": null,
"process_docs": null,
"sampler": "first_n",
"samples": null,
"split": "dev",
"target_delimiter": " "
},
"fewshot_delimiter": "\n\n",
"fewshot_split": "dev",
"metadata": {
"config_source": "[redacted]",
"dtype": "bfloat16",
"gpu_memory_utilization": 0.82,
"pretrained": "[redacted]",
"tensor_parallel_size": 1,
"trust_remote_code": true,
"version": 1.0
},
"metric_list": [
{
"aggregation": "mean",
"higher_is_better": true,
"metric": "acc"
}
],
"num_fewshot": 0,
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"target_delimiter": " ",
"task": "mmlu_philosophy",
"task_alias": "philosophy",
"test_split": "test",
"unsafe_code": false
},
"mmlu_prehistory": {
"dataset_name": "prehistory",
"dataset_path": "cais/mmlu",
"description": "The following are multiple choice questions (with answers) about prehistory.\n\n",
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_config": {
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_delimiter": "\n\n",
"fewshot_indices": null,
"gen_prefix": null,
"process_docs": null,
"sampler": "first_n",
"samples": null,
"split": "dev",
"target_delimiter": " "
},
"fewshot_delimiter": "\n\n",
"fewshot_split": "dev",
"metadata": {
"config_source": "[redacted]",
"dtype": "bfloat16",
"gpu_memory_utilization": 0.82,
"pretrained": "[redacted]",
"tensor_parallel_size": 1,
"trust_remote_code": true,
"version": 1.0
},
"metric_list": [
{
"aggregation": "mean",
"higher_is_better": true,
"metric": "acc"
}
],
"num_fewshot": 0,
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"target_delimiter": " ",
"task": "mmlu_prehistory",
"task_alias": "prehistory",
"test_split": "test",
"unsafe_code": false
},
"mmlu_professional_accounting": {
"dataset_name": "professional_accounting",
"dataset_path": "cais/mmlu",
"description": "The following are multiple choice questions (with answers) about professional accounting.\n\n",
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_config": {
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_delimiter": "\n\n",
"fewshot_indices": null,
"gen_prefix": null,
"process_docs": null,
"sampler": "first_n",
"samples": null,
"split": "dev",
"target_delimiter": " "
},
"fewshot_delimiter": "\n\n",
"fewshot_split": "dev",
"metadata": {
"config_source": "[redacted]",
"dtype": "bfloat16",
"gpu_memory_utilization": 0.82,
"pretrained": "[redacted]",
"tensor_parallel_size": 1,
"trust_remote_code": true,
"version": 1.0
},
"metric_list": [
{
"aggregation": "mean",
"higher_is_better": true,
"metric": "acc"
}
],
"num_fewshot": 0,
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"target_delimiter": " ",
"task": "mmlu_professional_accounting",
"task_alias": "professional_accounting",
"test_split": "test",
"unsafe_code": false
},
"mmlu_professional_law": {
"dataset_name": "professional_law",
"dataset_path": "cais/mmlu",
"description": "The following are multiple choice questions (with answers) about professional law.\n\n",
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_config": {
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_delimiter": "\n\n",
"fewshot_indices": null,
"gen_prefix": null,
"process_docs": null,
"sampler": "first_n",
"samples": null,
"split": "dev",
"target_delimiter": " "
},
"fewshot_delimiter": "\n\n",
"fewshot_split": "dev",
"metadata": {
"config_source": "[redacted]",
"dtype": "bfloat16",
"gpu_memory_utilization": 0.82,
"pretrained": "[redacted]",
"tensor_parallel_size": 1,
"trust_remote_code": true,
"version": 1.0
},
"metric_list": [
{
"aggregation": "mean",
"higher_is_better": true,
"metric": "acc"
}
],
"num_fewshot": 0,
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"target_delimiter": " ",
"task": "mmlu_professional_law",
"task_alias": "professional_law",
"test_split": "test",
"unsafe_code": false
},
"mmlu_professional_medicine": {
"dataset_name": "professional_medicine",
"dataset_path": "cais/mmlu",
"description": "The following are multiple choice questions (with answers) about professional medicine.\n\n",
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_config": {
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_delimiter": "\n\n",
"fewshot_indices": null,
"gen_prefix": null,
"process_docs": null,
"sampler": "first_n",
"samples": null,
"split": "dev",
"target_delimiter": " "
},
"fewshot_delimiter": "\n\n",
"fewshot_split": "dev",
"metadata": {
"config_source": "[redacted]",
"dtype": "bfloat16",
"gpu_memory_utilization": 0.82,
"pretrained": "[redacted]",
"tensor_parallel_size": 1,
"trust_remote_code": true,
"version": 1.0
},
"metric_list": [
{
"aggregation": "mean",
"higher_is_better": true,
"metric": "acc"
}
],
"num_fewshot": 0,
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"target_delimiter": " ",
"task": "mmlu_professional_medicine",
"task_alias": "professional_medicine",
"test_split": "test",
"unsafe_code": false
},
"mmlu_professional_psychology": {
"dataset_name": "professional_psychology",
"dataset_path": "cais/mmlu",
"description": "The following are multiple choice questions (with answers) about professional psychology.\n\n",
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_config": {
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_delimiter": "\n\n",
"fewshot_indices": null,
"gen_prefix": null,
"process_docs": null,
"sampler": "first_n",
"samples": null,
"split": "dev",
"target_delimiter": " "
},
"fewshot_delimiter": "\n\n",
"fewshot_split": "dev",
"metadata": {
"config_source": "[redacted]",
"dtype": "bfloat16",
"gpu_memory_utilization": 0.82,
"pretrained": "[redacted]",
"tensor_parallel_size": 1,
"trust_remote_code": true,
"version": 1.0
},
"metric_list": [
{
"aggregation": "mean",
"higher_is_better": true,
"metric": "acc"
}
],
"num_fewshot": 0,
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"target_delimiter": " ",
"task": "mmlu_professional_psychology",
"task_alias": "professional_psychology",
"test_split": "test",
"unsafe_code": false
},
"mmlu_public_relations": {
"dataset_name": "public_relations",
"dataset_path": "cais/mmlu",
"description": "The following are multiple choice questions (with answers) about public relations.\n\n",
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_config": {
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_delimiter": "\n\n",
"fewshot_indices": null,
"gen_prefix": null,
"process_docs": null,
"sampler": "first_n",
"samples": null,
"split": "dev",
"target_delimiter": " "
},
"fewshot_delimiter": "\n\n",
"fewshot_split": "dev",
"metadata": {
"config_source": "[redacted]",
"dtype": "bfloat16",
"gpu_memory_utilization": 0.82,
"pretrained": "[redacted]",
"tensor_parallel_size": 1,
"trust_remote_code": true,
"version": 1.0
},
"metric_list": [
{
"aggregation": "mean",
"higher_is_better": true,
"metric": "acc"
}
],
"num_fewshot": 0,
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"target_delimiter": " ",
"task": "mmlu_public_relations",
"task_alias": "public_relations",
"test_split": "test",
"unsafe_code": false
},
"mmlu_security_studies": {
"dataset_name": "security_studies",
"dataset_path": "cais/mmlu",
"description": "The following are multiple choice questions (with answers) about security studies.\n\n",
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_config": {
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_delimiter": "\n\n",
"fewshot_indices": null,
"gen_prefix": null,
"process_docs": null,
"sampler": "first_n",
"samples": null,
"split": "dev",
"target_delimiter": " "
},
"fewshot_delimiter": "\n\n",
"fewshot_split": "dev",
"metadata": {
"config_source": "[redacted]",
"dtype": "bfloat16",
"gpu_memory_utilization": 0.82,
"pretrained": "[redacted]",
"tensor_parallel_size": 1,
"trust_remote_code": true,
"version": 1.0
},
"metric_list": [
{
"aggregation": "mean",
"higher_is_better": true,
"metric": "acc"
}
],
"num_fewshot": 0,
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"target_delimiter": " ",
"task": "mmlu_security_studies",
"task_alias": "security_studies",
"test_split": "test",
"unsafe_code": false
},
"mmlu_sociology": {
"dataset_name": "sociology",
"dataset_path": "cais/mmlu",
"description": "The following are multiple choice questions (with answers) about sociology.\n\n",
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_config": {
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_delimiter": "\n\n",
"fewshot_indices": null,
"gen_prefix": null,
"process_docs": null,
"sampler": "first_n",
"samples": null,
"split": "dev",
"target_delimiter": " "
},
"fewshot_delimiter": "\n\n",
"fewshot_split": "dev",
"metadata": {
"config_source": "[redacted]",
"dtype": "bfloat16",
"gpu_memory_utilization": 0.82,
"pretrained": "[redacted]",
"tensor_parallel_size": 1,
"trust_remote_code": true,
"version": 1.0
},
"metric_list": [
{
"aggregation": "mean",
"higher_is_better": true,
"metric": "acc"
}
],
"num_fewshot": 0,
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"target_delimiter": " ",
"task": "mmlu_sociology",
"task_alias": "sociology",
"test_split": "test",
"unsafe_code": false
},
"mmlu_us_foreign_policy": {
"dataset_name": "us_foreign_policy",
"dataset_path": "cais/mmlu",
"description": "The following are multiple choice questions (with answers) about us foreign policy.\n\n",
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_config": {
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_delimiter": "\n\n",
"fewshot_indices": null,
"gen_prefix": null,
"process_docs": null,
"sampler": "first_n",
"samples": null,
"split": "dev",
"target_delimiter": " "
},
"fewshot_delimiter": "\n\n",
"fewshot_split": "dev",
"metadata": {
"config_source": "[redacted]",
"dtype": "bfloat16",
"gpu_memory_utilization": 0.82,
"pretrained": "[redacted]",
"tensor_parallel_size": 1,
"trust_remote_code": true,
"version": 1.0
},
"metric_list": [
{
"aggregation": "mean",
"higher_is_better": true,
"metric": "acc"
}
],
"num_fewshot": 0,
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"target_delimiter": " ",
"task": "mmlu_us_foreign_policy",
"task_alias": "us_foreign_policy",
"test_split": "test",
"unsafe_code": false
},
"mmlu_virology": {
"dataset_name": "virology",
"dataset_path": "cais/mmlu",
"description": "The following are multiple choice questions (with answers) about virology.\n\n",
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_config": {
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_delimiter": "\n\n",
"fewshot_indices": null,
"gen_prefix": null,
"process_docs": null,
"sampler": "first_n",
"samples": null,
"split": "dev",
"target_delimiter": " "
},
"fewshot_delimiter": "\n\n",
"fewshot_split": "dev",
"metadata": {
"config_source": "[redacted]",
"dtype": "bfloat16",
"gpu_memory_utilization": 0.82,
"pretrained": "[redacted]",
"tensor_parallel_size": 1,
"trust_remote_code": true,
"version": 1.0
},
"metric_list": [
{
"aggregation": "mean",
"higher_is_better": true,
"metric": "acc"
}
],
"num_fewshot": 0,
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"target_delimiter": " ",
"task": "mmlu_virology",
"task_alias": "virology",
"test_split": "test",
"unsafe_code": false
},
"mmlu_world_religions": {
"dataset_name": "world_religions",
"dataset_path": "cais/mmlu",
"description": "The following are multiple choice questions (with answers) about world religions.\n\n",
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_config": {
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"doc_to_target": "answer",
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
"fewshot_delimiter": "\n\n",
"fewshot_indices": null,
"gen_prefix": null,
"process_docs": null,
"sampler": "first_n",
"samples": null,
"split": "dev",
"target_delimiter": " "
},
"fewshot_delimiter": "\n\n",
"fewshot_split": "dev",
"metadata": {
"config_source": "[redacted]",
"dtype": "bfloat16",
"gpu_memory_utilization": 0.82,
"pretrained": "[redacted]",
"tensor_parallel_size": 1,
"trust_remote_code": true,
"version": 1.0
},
"metric_list": [
{
"aggregation": "mean",
"higher_is_better": true,
"metric": "acc"
}
],
"num_fewshot": 0,
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"target_delimiter": " ",
"task": "mmlu_world_religions",
"task_alias": "world_religions",
"test_split": "test",
"unsafe_code": false
}
},
"date": 1781174539.5106196,
"eot_token_id": 151645,
"fewshot_as_multiturn": true,
"git_hash": null,
"group_subtasks": {
"mmlu": [
"mmlu_stem",
"mmlu_other",
"mmlu_social_sciences",
"mmlu_humanities"
],
"mmlu_humanities": [
"mmlu_formal_logic",
"mmlu_high_school_european_history",
"mmlu_high_school_us_history",
"mmlu_high_school_world_history",
"mmlu_international_law",
"mmlu_jurisprudence",
"mmlu_logical_fallacies",
"mmlu_moral_disputes",
"mmlu_moral_scenarios",
"mmlu_philosophy",
"mmlu_prehistory",
"mmlu_professional_law",
"mmlu_world_religions"
],
"mmlu_other": [
"mmlu_business_ethics",
"mmlu_clinical_knowledge",
"mmlu_college_medicine",
"mmlu_global_facts",
"mmlu_human_aging",
"mmlu_management",
"mmlu_marketing",
"mmlu_medical_genetics",
"mmlu_miscellaneous",
"mmlu_nutrition",
"mmlu_professional_accounting",
"mmlu_professional_medicine",
"mmlu_virology"
],
"mmlu_social_sciences": [
"mmlu_econometrics",
"mmlu_high_school_geography",
"mmlu_high_school_government_and_politics",
"mmlu_high_school_macroeconomics",
"mmlu_high_school_microeconomics",
"mmlu_high_school_psychology",
"mmlu_human_sexuality",
"mmlu_professional_psychology",
"mmlu_public_relations",
"mmlu_security_studies",
"mmlu_sociology",
"mmlu_us_foreign_policy"
],
"mmlu_stem": [
"mmlu_abstract_algebra",
"mmlu_anatomy",
"mmlu_astronomy",
"mmlu_college_biology",
"mmlu_college_chemistry",
"mmlu_college_computer_science",
"mmlu_college_mathematics",
"mmlu_college_physics",
"mmlu_computer_security",
"mmlu_conceptual_physics",
"mmlu_electrical_engineering",
"mmlu_elementary_mathematics",
"mmlu_high_school_biology",
"mmlu_high_school_chemistry",
"mmlu_high_school_computer_science",
"mmlu_high_school_mathematics",
"mmlu_high_school_physics",
"mmlu_high_school_statistics",
"mmlu_machine_learning"
]
},
"groups": {
"mmlu": {
"acc,none": 0.6575986326734083,
"acc_stderr,none": 0.0036785245967130517,
"alias": "mmlu",
"name": "mmlu",
"sample_count": {
"acc,none": 14042
},
"sample_len": 14042
},
"mmlu_humanities": {
"acc,none": 0.5883103081827843,
"acc_stderr,none": 0.006616678303159137,
"alias": "humanities",
"name": "mmlu_humanities",
"sample_count": {
"acc,none": 4705
},
"sample_len": 4705
},
"mmlu_other": {
"acc,none": 0.7219182491149019,
"acc_stderr,none": 0.007740746200087848,
"alias": "other",
"name": "mmlu_other",
"sample_count": {
"acc,none": 3107
},
"sample_len": 3107
},
"mmlu_social_sciences": {
"acc,none": 0.7897302567435814,
"acc_stderr,none": 0.007206549343813523,
"alias": "social sciences",
"name": "mmlu_social_sciences",
"sample_count": {
"acc,none": 3077
},
"sample_len": 3077
},
"mmlu_stem": {
"acc,none": 0.5686647637170948,
"acc_stderr,none": 0.007952947130384178,
"alias": "stem",
"name": "mmlu_stem",
"sample_count": {
"acc,none": 3153
},
"sample_len": 3153
}
},
"higher_is_better": {
"mmlu_abstract_algebra": {
"acc": true
},
"mmlu_anatomy": {
"acc": true
},
"mmlu_astronomy": {
"acc": true
},
"mmlu_business_ethics": {
"acc": true
},
"mmlu_clinical_knowledge": {
"acc": true
},
"mmlu_college_biology": {
"acc": true
},
"mmlu_college_chemistry": {
"acc": true
},
"mmlu_college_computer_science": {
"acc": true
},
"mmlu_college_mathematics": {
"acc": true
},
"mmlu_college_medicine": {
"acc": true
},
"mmlu_college_physics": {
"acc": true
},
"mmlu_computer_security": {
"acc": true
},
"mmlu_conceptual_physics": {
"acc": true
},
"mmlu_econometrics": {
"acc": true
},
"mmlu_electrical_engineering": {
"acc": true
},
"mmlu_elementary_mathematics": {
"acc": true
},
"mmlu_formal_logic": {
"acc": true
},
"mmlu_global_facts": {
"acc": true
},
"mmlu_high_school_biology": {
"acc": true
},
"mmlu_high_school_chemistry": {
"acc": true
},
"mmlu_high_school_computer_science": {
"acc": true
},
"mmlu_high_school_european_history": {
"acc": true
},
"mmlu_high_school_geography": {
"acc": true
},
"mmlu_high_school_government_and_politics": {
"acc": true
},
"mmlu_high_school_macroeconomics": {
"acc": true
},
"mmlu_high_school_mathematics": {
"acc": true
},
"mmlu_high_school_microeconomics": {
"acc": true
},
"mmlu_high_school_physics": {
"acc": true
},
"mmlu_high_school_psychology": {
"acc": true
},
"mmlu_high_school_statistics": {
"acc": true
},
"mmlu_high_school_us_history": {
"acc": true
},
"mmlu_high_school_world_history": {
"acc": true
},
"mmlu_human_aging": {
"acc": true
},
"mmlu_human_sexuality": {
"acc": true
},
"mmlu_humanities": {
"acc": true
},
"mmlu_international_law": {
"acc": true
},
"mmlu_jurisprudence": {
"acc": true
},
"mmlu_logical_fallacies": {
"acc": true
},
"mmlu_machine_learning": {
"acc": true
},
"mmlu_management": {
"acc": true
},
"mmlu_marketing": {
"acc": true
},
"mmlu_medical_genetics": {
"acc": true
},
"mmlu_miscellaneous": {
"acc": true
},
"mmlu_moral_disputes": {
"acc": true
},
"mmlu_moral_scenarios": {
"acc": true
},
"mmlu_nutrition": {
"acc": true
},
"mmlu_other": {
"acc": true
},
"mmlu_philosophy": {
"acc": true
},
"mmlu_prehistory": {
"acc": true
},
"mmlu_professional_accounting": {
"acc": true
},
"mmlu_professional_law": {
"acc": true
},
"mmlu_professional_medicine": {
"acc": true
},
"mmlu_professional_psychology": {
"acc": true
},
"mmlu_public_relations": {
"acc": true
},
"mmlu_security_studies": {
"acc": true
},
"mmlu_social_sciences": {
"acc": true
},
"mmlu_sociology": {
"acc": true
},
"mmlu_stem": {
"acc": true
},
"mmlu_us_foreign_policy": {
"acc": true
},
"mmlu_virology": {
"acc": true
},
"mmlu_world_religions": {
"acc": true
}
},
"lm_eval_version": "0.4.12",
"max_length": 262144,
"model_name": "kosa-labs/kosa-4B-it-v1",
"model_name_sanitized": "kosa-labs/kosa-4B-it-v1",
"model_source": "kosa-labs/kosa-4B-it-v1",
"n-samples": {
"mmlu_abstract_algebra": {
"effective": 100,
"original": 100
},
"mmlu_anatomy": {
"effective": 135,
"original": 135
},
"mmlu_astronomy": {
"effective": 152,
"original": 152
},
"mmlu_business_ethics": {
"effective": 100,
"original": 100
},
"mmlu_clinical_knowledge": {
"effective": 265,
"original": 265
},
"mmlu_college_biology": {
"effective": 144,
"original": 144
},
"mmlu_college_chemistry": {
"effective": 100,
"original": 100
},
"mmlu_college_computer_science": {
"effective": 100,
"original": 100
},
"mmlu_college_mathematics": {
"effective": 100,
"original": 100
},
"mmlu_college_medicine": {
"effective": 173,
"original": 173
},
"mmlu_college_physics": {
"effective": 102,
"original": 102
},
"mmlu_computer_security": {
"effective": 100,
"original": 100
},
"mmlu_conceptual_physics": {
"effective": 235,
"original": 235
},
"mmlu_econometrics": {
"effective": 114,
"original": 114
},
"mmlu_electrical_engineering": {
"effective": 145,
"original": 145
},
"mmlu_elementary_mathematics": {
"effective": 378,
"original": 378
},
"mmlu_formal_logic": {
"effective": 126,
"original": 126
},
"mmlu_global_facts": {
"effective": 100,
"original": 100
},
"mmlu_high_school_biology": {
"effective": 310,
"original": 310
},
"mmlu_high_school_chemistry": {
"effective": 203,
"original": 203
},
"mmlu_high_school_computer_science": {
"effective": 100,
"original": 100
},
"mmlu_high_school_european_history": {
"effective": 165,
"original": 165
},
"mmlu_high_school_geography": {
"effective": 198,
"original": 198
},
"mmlu_high_school_government_and_politics": {
"effective": 193,
"original": 193
},
"mmlu_high_school_macroeconomics": {
"effective": 390,
"original": 390
},
"mmlu_high_school_mathematics": {
"effective": 270,
"original": 270
},
"mmlu_high_school_microeconomics": {
"effective": 238,
"original": 238
},
"mmlu_high_school_physics": {
"effective": 151,
"original": 151
},
"mmlu_high_school_psychology": {
"effective": 545,
"original": 545
},
"mmlu_high_school_statistics": {
"effective": 216,
"original": 216
},
"mmlu_high_school_us_history": {
"effective": 204,
"original": 204
},
"mmlu_high_school_world_history": {
"effective": 237,
"original": 237
},
"mmlu_human_aging": {
"effective": 223,
"original": 223
},
"mmlu_human_sexuality": {
"effective": 131,
"original": 131
},
"mmlu_international_law": {
"effective": 121,
"original": 121
},
"mmlu_jurisprudence": {
"effective": 108,
"original": 108
},
"mmlu_logical_fallacies": {
"effective": 163,
"original": 163
},
"mmlu_machine_learning": {
"effective": 112,
"original": 112
},
"mmlu_management": {
"effective": 103,
"original": 103
},
"mmlu_marketing": {
"effective": 234,
"original": 234
},
"mmlu_medical_genetics": {
"effective": 100,
"original": 100
},
"mmlu_miscellaneous": {
"effective": 783,
"original": 783
},
"mmlu_moral_disputes": {
"effective": 346,
"original": 346
},
"mmlu_moral_scenarios": {
"effective": 895,
"original": 895
},
"mmlu_nutrition": {
"effective": 306,
"original": 306
},
"mmlu_philosophy": {
"effective": 311,
"original": 311
},
"mmlu_prehistory": {
"effective": 324,
"original": 324
},
"mmlu_professional_accounting": {
"effective": 282,
"original": 282
},
"mmlu_professional_law": {
"effective": 1534,
"original": 1534
},
"mmlu_professional_medicine": {
"effective": 272,
"original": 272
},
"mmlu_professional_psychology": {
"effective": 612,
"original": 612
},
"mmlu_public_relations": {
"effective": 110,
"original": 110
},
"mmlu_security_studies": {
"effective": 245,
"original": 245
},
"mmlu_sociology": {
"effective": 201,
"original": 201
},
"mmlu_us_foreign_policy": {
"effective": 100,
"original": 100
},
"mmlu_virology": {
"effective": 166,
"original": 166
},
"mmlu_world_religions": {
"effective": 171,
"original": 171
}
},
"n-shot": {
"mmlu_abstract_algebra": 0,
"mmlu_anatomy": 0,
"mmlu_astronomy": 0,
"mmlu_business_ethics": 0,
"mmlu_clinical_knowledge": 0,
"mmlu_college_biology": 0,
"mmlu_college_chemistry": 0,
"mmlu_college_computer_science": 0,
"mmlu_college_mathematics": 0,
"mmlu_college_medicine": 0,
"mmlu_college_physics": 0,
"mmlu_computer_security": 0,
"mmlu_conceptual_physics": 0,
"mmlu_econometrics": 0,
"mmlu_electrical_engineering": 0,
"mmlu_elementary_mathematics": 0,
"mmlu_formal_logic": 0,
"mmlu_global_facts": 0,
"mmlu_high_school_biology": 0,
"mmlu_high_school_chemistry": 0,
"mmlu_high_school_computer_science": 0,
"mmlu_high_school_european_history": 0,
"mmlu_high_school_geography": 0,
"mmlu_high_school_government_and_politics": 0,
"mmlu_high_school_macroeconomics": 0,
"mmlu_high_school_mathematics": 0,
"mmlu_high_school_microeconomics": 0,
"mmlu_high_school_physics": 0,
"mmlu_high_school_psychology": 0,
"mmlu_high_school_statistics": 0,
"mmlu_high_school_us_history": 0,
"mmlu_high_school_world_history": 0,
"mmlu_human_aging": 0,
"mmlu_human_sexuality": 0,
"mmlu_humanities": 0,
"mmlu_international_law": 0,
"mmlu_jurisprudence": 0,
"mmlu_logical_fallacies": 0,
"mmlu_machine_learning": 0,
"mmlu_management": 0,
"mmlu_marketing": 0,
"mmlu_medical_genetics": 0,
"mmlu_miscellaneous": 0,
"mmlu_moral_disputes": 0,
"mmlu_moral_scenarios": 0,
"mmlu_nutrition": 0,
"mmlu_other": 0,
"mmlu_philosophy": 0,
"mmlu_prehistory": 0,
"mmlu_professional_accounting": 0,
"mmlu_professional_law": 0,
"mmlu_professional_medicine": 0,
"mmlu_professional_psychology": 0,
"mmlu_public_relations": 0,
"mmlu_security_studies": 0,
"mmlu_social_sciences": 0,
"mmlu_sociology": 0,
"mmlu_stem": 0,
"mmlu_us_foreign_policy": 0,
"mmlu_virology": 0,
"mmlu_world_religions": 0
},
"results": {
"mmlu": {
"acc,none": 0.6575986326734083,
"acc_stderr,none": 0.0036785245967130517,
"alias": "mmlu",
"name": "mmlu",
"sample_count": {
"acc,none": 14042
},
"sample_len": 14042
},
"mmlu_abstract_algebra": {
"acc,none": 0.25,
"acc_stderr,none": 0.04351941398892446,
"alias": "abstract_algebra",
"name": "mmlu_abstract_algebra",
"sample_len": 100
},
"mmlu_anatomy": {
"acc,none": 0.674074074074074,
"acc_stderr,none": 0.040491220417025006,
"alias": "anatomy",
"name": "mmlu_anatomy",
"sample_len": 135
},
"mmlu_astronomy": {
"acc,none": 0.8355263157894737,
"acc_stderr,none": 0.030167533468632726,
"alias": "astronomy",
"name": "mmlu_astronomy",
"sample_len": 152
},
"mmlu_business_ethics": {
"acc,none": 0.74,
"acc_stderr,none": 0.0440844002276808,
"alias": "business_ethics",
"name": "mmlu_business_ethics",
"sample_len": 100
},
"mmlu_clinical_knowledge": {
"acc,none": 0.7509433962264151,
"acc_stderr,none": 0.02661648298050167,
"alias": "clinical_knowledge",
"name": "mmlu_clinical_knowledge",
"sample_len": 265
},
"mmlu_college_biology": {
"acc,none": 0.8680555555555556,
"acc_stderr,none": 0.02830096838204442,
"alias": "college_biology",
"name": "mmlu_college_biology",
"sample_len": 144
},
"mmlu_college_chemistry": {
"acc,none": 0.47,
"acc_stderr,none": 0.05016135580465919,
"alias": "college_chemistry",
"name": "mmlu_college_chemistry",
"sample_len": 100
},
"mmlu_college_computer_science": {
"acc,none": 0.51,
"acc_stderr,none": 0.05024183937956913,
"alias": "college_computer_science",
"name": "mmlu_college_computer_science",
"sample_len": 100
},
"mmlu_college_mathematics": {
"acc,none": 0.24,
"acc_stderr,none": 0.04292346959909278,
"alias": "college_mathematics",
"name": "mmlu_college_mathematics",
"sample_len": 100
},
"mmlu_college_medicine": {
"acc,none": 0.6705202312138728,
"acc_stderr,none": 0.03583901754736415,
"alias": "college_medicine",
"name": "mmlu_college_medicine",
"sample_len": 173
},
"mmlu_college_physics": {
"acc,none": 0.39215686274509803,
"acc_stderr,none": 0.04858083574266346,
"alias": "college_physics",
"name": "mmlu_college_physics",
"sample_len": 102
},
"mmlu_computer_security": {
"acc,none": 0.79,
"acc_stderr,none": 0.040936018074033236,
"alias": "computer_security",
"name": "mmlu_computer_security",
"sample_len": 100
},
"mmlu_conceptual_physics": {
"acc,none": 0.774468085106383,
"acc_stderr,none": 0.02732107841738755,
"alias": "conceptual_physics",
"name": "mmlu_conceptual_physics",
"sample_len": 235
},
"mmlu_econometrics": {
"acc,none": 0.5614035087719298,
"acc_stderr,none": 0.04668000738510451,
"alias": "econometrics",
"name": "mmlu_econometrics",
"sample_len": 114
},
"mmlu_electrical_engineering": {
"acc,none": 0.6896551724137931,
"acc_stderr,none": 0.03855289616378948,
"alias": "electrical_engineering",
"name": "mmlu_electrical_engineering",
"sample_len": 145
},
"mmlu_elementary_mathematics": {
"acc,none": 0.36507936507936506,
"acc_stderr,none": 0.02479606060269989,
"alias": "elementary_mathematics",
"name": "mmlu_elementary_mathematics",
"sample_len": 378
},
"mmlu_formal_logic": {
"acc,none": 0.5079365079365079,
"acc_stderr,none": 0.04471572536294351,
"alias": "formal_logic",
"name": "mmlu_formal_logic",
"sample_len": 126
},
"mmlu_global_facts": {
"acc,none": 0.43,
"acc_stderr,none": 0.049756985195624305,
"alias": "global_facts",
"name": "mmlu_global_facts",
"sample_len": 100
},
"mmlu_high_school_biology": {
"acc,none": 0.8516129032258064,
"acc_stderr,none": 0.02022273755433042,
"alias": "high_school_biology",
"name": "mmlu_high_school_biology",
"sample_len": 310
},
"mmlu_high_school_chemistry": {
"acc,none": 0.6009852216748769,
"acc_stderr,none": 0.03445487686264716,
"alias": "high_school_chemistry",
"name": "mmlu_high_school_chemistry",
"sample_len": 203
},
"mmlu_high_school_computer_science": {
"acc,none": 0.78,
"acc_stderr,none": 0.041633319989322654,
"alias": "high_school_computer_science",
"name": "mmlu_high_school_computer_science",
"sample_len": 100
},
"mmlu_high_school_european_history": {
"acc,none": 0.8363636363636363,
"acc_stderr,none": 0.028887872395487978,
"alias": "high_school_european_history",
"name": "mmlu_high_school_european_history",
"sample_len": 165
},
"mmlu_high_school_geography": {
"acc,none": 0.8434343434343434,
"acc_stderr,none": 0.025890520358141458,
"alias": "high_school_geography",
"name": "mmlu_high_school_geography",
"sample_len": 198
},
"mmlu_high_school_government_and_politics": {
"acc,none": 0.8911917098445595,
"acc_stderr,none": 0.02247325333276875,
"alias": "high_school_government_and_politics",
"name": "mmlu_high_school_government_and_politics",
"sample_len": 193
},
"mmlu_high_school_macroeconomics": {
"acc,none": 0.7256410256410256,
"acc_stderr,none": 0.022622765767493235,
"alias": "high_school_macroeconomics",
"name": "mmlu_high_school_macroeconomics",
"sample_len": 390
},
"mmlu_high_school_mathematics": {
"acc,none": 0.21851851851851853,
"acc_stderr,none": 0.025195752251823727,
"alias": "high_school_mathematics",
"name": "mmlu_high_school_mathematics",
"sample_len": 270
},
"mmlu_high_school_microeconomics": {
"acc,none": 0.8445378151260504,
"acc_stderr,none": 0.023536818625398935,
"alias": "high_school_microeconomics",
"name": "mmlu_high_school_microeconomics",
"sample_len": 238
},
"mmlu_high_school_physics": {
"acc,none": 0.4768211920529801,
"acc_stderr,none": 0.04078093859163086,
"alias": "high_school_physics",
"name": "mmlu_high_school_physics",
"sample_len": 151
},
"mmlu_high_school_psychology": {
"acc,none": 0.8880733944954129,
"acc_stderr,none": 0.013517352714958736,
"alias": "high_school_psychology",
"name": "mmlu_high_school_psychology",
"sample_len": 545
},
"mmlu_high_school_statistics": {
"acc,none": 0.49537037037037035,
"acc_stderr,none": 0.034098255191635805,
"alias": "high_school_statistics",
"name": "mmlu_high_school_statistics",
"sample_len": 216
},
"mmlu_high_school_us_history": {
"acc,none": 0.8529411764705882,
"acc_stderr,none": 0.024857478080250486,
"alias": "high_school_us_history",
"name": "mmlu_high_school_us_history",
"sample_len": 204
},
"mmlu_high_school_world_history": {
"acc,none": 0.8481012658227848,
"acc_stderr,none": 0.023363878096632425,
"alias": "high_school_world_history",
"name": "mmlu_high_school_world_history",
"sample_len": 237
},
"mmlu_human_aging": {
"acc,none": 0.6860986547085202,
"acc_stderr,none": 0.031146796482972486,
"alias": "human_aging",
"name": "mmlu_human_aging",
"sample_len": 223
},
"mmlu_human_sexuality": {
"acc,none": 0.7480916030534351,
"acc_stderr,none": 0.03807387116306089,
"alias": "human_sexuality",
"name": "mmlu_human_sexuality",
"sample_len": 131
},
"mmlu_humanities": {
"acc,none": 0.5883103081827843,
"acc_stderr,none": 0.006616678303159137,
"alias": "humanities",
"name": "mmlu_humanities",
"sample_count": {
"acc,none": 4705
},
"sample_len": 4705
},
"mmlu_international_law": {
"acc,none": 0.7933884297520661,
"acc_stderr,none": 0.03695980128098826,
"alias": "international_law",
"name": "mmlu_international_law",
"sample_len": 121
},
"mmlu_jurisprudence": {
"acc,none": 0.7777777777777778,
"acc_stderr,none": 0.04019107472557348,
"alias": "jurisprudence",
"name": "mmlu_jurisprudence",
"sample_len": 108
},
"mmlu_logical_fallacies": {
"acc,none": 0.7975460122699386,
"acc_stderr,none": 0.031570650789119054,
"alias": "logical_fallacies",
"name": "mmlu_logical_fallacies",
"sample_len": 163
},
"mmlu_machine_learning": {
"acc,none": 0.5535714285714286,
"acc_stderr,none": 0.04718471485219583,
"alias": "machine_learning",
"name": "mmlu_machine_learning",
"sample_len": 112
},
"mmlu_management": {
"acc,none": 0.8446601941747572,
"acc_stderr,none": 0.03586594738573976,
"alias": "management",
"name": "mmlu_management",
"sample_len": 103
},
"mmlu_marketing": {
"acc,none": 0.8888888888888888,
"acc_stderr,none": 0.02058849131609242,
"alias": "marketing",
"name": "mmlu_marketing",
"sample_len": 234
},
"mmlu_medical_genetics": {
"acc,none": 0.83,
"acc_stderr,none": 0.03775251680686369,
"alias": "medical_genetics",
"name": "mmlu_medical_genetics",
"sample_len": 100
},
"mmlu_miscellaneous": {
"acc,none": 0.8071519795657727,
"acc_stderr,none": 0.014108533515757481,
"alias": "miscellaneous",
"name": "mmlu_miscellaneous",
"sample_len": 783
},
"mmlu_moral_disputes": {
"acc,none": 0.7167630057803468,
"acc_stderr,none": 0.024257901705323326,
"alias": "moral_disputes",
"name": "mmlu_moral_disputes",
"sample_len": 346
},
"mmlu_moral_scenarios": {
"acc,none": 0.31508379888268156,
"acc_stderr,none": 0.015536850852473702,
"alias": "moral_scenarios",
"name": "mmlu_moral_scenarios",
"sample_len": 895
},
"mmlu_nutrition": {
"acc,none": 0.7810457516339869,
"acc_stderr,none": 0.02367908986180772,
"alias": "nutrition",
"name": "mmlu_nutrition",
"sample_len": 306
},
"mmlu_other": {
"acc,none": 0.7219182491149019,
"acc_stderr,none": 0.007740746200087848,
"alias": "other",
"name": "mmlu_other",
"sample_count": {
"acc,none": 3107
},
"sample_len": 3107
},
"mmlu_philosophy": {
"acc,none": 0.7106109324758842,
"acc_stderr,none": 0.02575586592263286,
"alias": "philosophy",
"name": "mmlu_philosophy",
"sample_len": 311
},
"mmlu_prehistory": {
"acc,none": 0.7716049382716049,
"acc_stderr,none": 0.02335821184062624,
"alias": "prehistory",
"name": "mmlu_prehistory",
"sample_len": 324
},
"mmlu_professional_accounting": {
"acc,none": 0.49645390070921985,
"acc_stderr,none": 0.029826749153280853,
"alias": "professional_accounting",
"name": "mmlu_professional_accounting",
"sample_len": 282
},
"mmlu_professional_law": {
"acc,none": 0.4810951760104302,
"acc_stderr,none": 0.012761104871472521,
"alias": "professional_law",
"name": "mmlu_professional_law",
"sample_len": 1534
},
"mmlu_professional_medicine": {
"acc,none": 0.6764705882352942,
"acc_stderr,none": 0.028418208619406728,
"alias": "professional_medicine",
"name": "mmlu_professional_medicine",
"sample_len": 272
},
"mmlu_professional_psychology": {
"acc,none": 0.7336601307189542,
"acc_stderr,none": 0.017883188134667258,
"alias": "professional_psychology",
"name": "mmlu_professional_psychology",
"sample_len": 612
},
"mmlu_public_relations": {
"acc,none": 0.6545454545454545,
"acc_stderr,none": 0.045546196175410524,
"alias": "public_relations",
"name": "mmlu_public_relations",
"sample_len": 110
},
"mmlu_security_studies": {
"acc,none": 0.7673469387755102,
"acc_stderr,none": 0.02704925791589615,
"alias": "security_studies",
"name": "mmlu_security_studies",
"sample_len": 245
},
"mmlu_social_sciences": {
"acc,none": 0.7897302567435814,
"acc_stderr,none": 0.007206549343813523,
"alias": "social sciences",
"name": "mmlu_social_sciences",
"sample_count": {
"acc,none": 3077
},
"sample_len": 3077
},
"mmlu_sociology": {
"acc,none": 0.8507462686567164,
"acc_stderr,none": 0.02519692987482706,
"alias": "sociology",
"name": "mmlu_sociology",
"sample_len": 201
},
"mmlu_stem": {
"acc,none": 0.5686647637170948,
"acc_stderr,none": 0.007952947130384178,
"alias": "stem",
"name": "mmlu_stem",
"sample_count": {
"acc,none": 3153
},
"sample_len": 3153
},
"mmlu_us_foreign_policy": {
"acc,none": 0.81,
"acc_stderr,none": 0.039427724440366255,
"alias": "us_foreign_policy",
"name": "mmlu_us_foreign_policy",
"sample_len": 100
},
"mmlu_virology": {
"acc,none": 0.5120481927710844,
"acc_stderr,none": 0.038913644958358196,
"alias": "virology",
"name": "mmlu_virology",
"sample_len": 166
},
"mmlu_world_religions": {
"acc,none": 0.8304093567251462,
"acc_stderr,none": 0.02878210810540169,
"alias": "world_religions",
"name": "mmlu_world_religions",
"sample_len": 171
}
},
"system_instruction": null,
"system_instruction_sha": null,
"task_hashes": {
"mmlu_abstract_algebra": "1ecc29cafeff483bf80347adc2878dae00d01596988032f438afce6291c8cfe7",
"mmlu_anatomy": "51232e78f32967e0dd431bab9309f69be490f9998b4ddb93b641b09ba1f613cf",
"mmlu_astronomy": "3e8d7c33bd0431518a3d5d4b1666bdabccf46c4cc4cfe95a3886cb2266743f86",
"mmlu_business_ethics": "6ec93578c18c7bba58a384ecaa7e6e84c5e681262b34b1caa76abbbb0f9ebb02",
"mmlu_clinical_knowledge": "cf39fec90aa76a379e440c1f0d5f5b04d9f32edc4004ffcfb8eadcc25c18b118",
"mmlu_college_biology": "5378a297efa1cd71344228c95dcb29ba54757c7e527261400a4d65fd8ceed92b",
"mmlu_college_chemistry": "714e60ea7e340f0344ffde41b247a19b68e6384152a1bdc9acc0ecf24cd0e7cd",
"mmlu_college_computer_science": "2b66d244f98e61e6a78e5ed48b728e0ca2ca81c7757ccdd086fe6b0c1cf02da1",
"mmlu_college_mathematics": "90b1ce25d7d38abd925181ae3aad095cfd783375fe8de0607a57420f47454f0f",
"mmlu_college_medicine": "00cc41d26670913bdee6181e5b602726dde9f03e6a43236c2bf57486149845be",
"mmlu_college_physics": "2abddb12a247904f9ba60b39766b19a8967c24fe8eee021022a353b93d73742c",
"mmlu_computer_security": "8ffba21cebb9e39d5794c848a22e7e37cddff5c59f7b47a1d89872d9e8704b0c",
"mmlu_conceptual_physics": "c0da3219fe7e5292dad2aa7e72790e3eba951711007cc57618b0850d941b85ec",
"mmlu_econometrics": "bc667156b739500e02f7c8cb8f9621b6d62ea2011e1eaa3404cb45f6d16778ec",
"mmlu_electrical_engineering": "0a02cb16a31ec7debba4190ab1cc31afadb9416764e0b074b9b96553f346b892",
"mmlu_elementary_mathematics": "6048f1506468e757010d537a0ceedaadc587c18159a03e7664b5994843afedc2",
"mmlu_formal_logic": "5fef452fe37d3c5aefce6276a26a839e5ee5d655d72adf50bbf0a8ae0e8d74e4",
"mmlu_global_facts": "6163d6678b902c8cb5cd6c50f59bc0f31a9a356e977fcd6184614f48e3320e7a",
"mmlu_high_school_biology": "2aabe83bb100a352d24a0136c24e612b68490bbe818c33f06cc6cee5379ecad8",
"mmlu_high_school_chemistry": "dd3321fd42c8717094f57dd0eaeab0002d9ed192518a7de7aaef4913796c30f0",
"mmlu_high_school_computer_science": "888aabb9a7ac77299e8b2b8429fa44c1fbbb5ac8165f556a0b00b347e51de3b5",
"mmlu_high_school_european_history": "0b7be1963c7e224bffc18d72fe277408c4980f854a5fc94805fc008133640f44",
"mmlu_high_school_geography": "915db5f9f9c584c6886ae92cfc28ad9c364bf685f22c15009ad8bf064b4a8027",
"mmlu_high_school_government_and_politics": "411c6c6c179d1e2a252fab88f33123721540fc70f8223310e1f1349ec0b67753",
"mmlu_high_school_macroeconomics": "291b29471838255d1a2dd704960610e98b7e3f2fd001e3aabc6b25be0c236e29",
"mmlu_high_school_mathematics": "9b9567fc5c7875065ac91eac69621904f201cab822621440a2442a390f62b497",
"mmlu_high_school_microeconomics": "6f4a887cb73beee5414248002f75216c16d7c8ff739b37a2607c6bcfad16712b",
"mmlu_high_school_physics": "53edb93aefb6fc54fba283278bae10a58a7902902ff55edc30cb9c81745dbaa8",
"mmlu_high_school_psychology": "ce01f01ff9a3bdc3e956ada7ec7c02f13771c386d076323056ac00eec0095785",
"mmlu_high_school_statistics": "e0d652bf0cb6f927b0fc738e58a141f0ba8f9073615f4ddd70bf7a3b97998f21",
"mmlu_high_school_us_history": "1b31fdc7e0d51803fa6107a0c7aa3b6968031bc449ed3513a7349956fa0b53d3",
"mmlu_high_school_world_history": "9e352460adb245947aaf9671f135ae6484e44a115312955c45c7caaa0688f160",
"mmlu_human_aging": "4b4276095e96dc6e2872517b2becc2ebe9fad0f42b321486f6f2ceaef81f385e",
"mmlu_human_sexuality": "9395014a475c63632383587439df5c48c5dd3333eaa8d10ef9e0eaabd0997c3a",
"mmlu_international_law": "3d2036ffe61014fa8a2de679c6bee2c0c3ade112f03cd55e4a6fba5bdc56bf7b",
"mmlu_jurisprudence": "1548c06ef17823048b473d028f4889b3cf014beac4ac808521898ee141a34d23",
"mmlu_logical_fallacies": "66f9691122f3fa813b86fda1131037c03cf42babe650c7374924f41882716332",
"mmlu_machine_learning": "13ad1d3650a26d655a2a2eaa3be1f106d990e5b693b0e9006f8ebef221d2ed82",
"mmlu_management": "84a18a4a3039f23b7dd18838c89a80171025a1a45eda3305464a3cfcd8257b69",
"mmlu_marketing": "d6382d405e202221a78087356861723acb445fbd6f1db94152201f788b5c0606",
"mmlu_medical_genetics": "b4546cc241c5811dddfec47d0bba37122fbd65545c93d60f50b993e810d37e2d",
"mmlu_miscellaneous": "ecf4c3e0dc895ca5b7803d1b3f01fe3027b461f900ac001d36f5dd750c8eca1d",
"mmlu_moral_disputes": "84f580166c00063790619588b4f9fcd2f800a6a04778f08c1bada043bfdc6ec0",
"mmlu_moral_scenarios": "0dda4f43fd25e66311e64f4590698bfcc4c00c24ac58dbfec74b613512280ffc",
"mmlu_nutrition": "732959f39ac0c6c0e9c3823082d3669467531cebfb5f7550d79bcc31e5cfdbce",
"mmlu_philosophy": "ff6284cd5c6bfdb17898ef97dabc97916b70fc7485c57543edabdd44b1d8b543",
"mmlu_prehistory": "efb58ea420f5b7975ff34766ddc7470676e1ababa0b4705bfdf150f6871fdd83",
"mmlu_professional_accounting": "3c555bc35f86d097f509755f487781d2bd288de5dfe9aa59d8009a1dd0583c45",
"mmlu_professional_law": "b108c04f61cbbb4b9351de3730d7e3396710c7702c6cf75703155dbfdbc7819d",
"mmlu_professional_medicine": "0821fa48ca3715604bbef122215aea4e55f218dad6ba5191da2eb831971ac6f9",
"mmlu_professional_psychology": "ab273adb58e2b365b3811e4ed4c6485c9630f78058580273703eb2573dde9010",
"mmlu_public_relations": "da7dfa166f545d33a61c930908eee4f8d8544cad6328569b1c4b341becaf68f1",
"mmlu_security_studies": "85ea8f6c3b791998f3c1d57164b6e28bcf4498f65fc1988dfa4a6a65d31fb505",
"mmlu_sociology": "055d024e9587edc811c54e6cdd8fd74d535c6f5b1aad17cfd5f1598a1371606a",
"mmlu_us_foreign_policy": "a99e6492b6831a5df8c7f4c76edf15b500e59ab4c87dc73fdbd61d43bc704682",
"mmlu_virology": "ba4fc492ed4e9af97e30850257d2fdc5e7b53b37e7b4d68d443f7bd157aac21f",
"mmlu_world_religions": "9634e441f5f53329b4b6c760b08b1a130f8f03b3cc1902b23d07f57db76b7263"
},
"tokenizer_bos_token": [
null,
"None"
],
"tokenizer_eos_token": [
"<|im_end|>",
"151645"
],
"tokenizer_pad_token": [
"<|endoftext|>",
"151643"
],
"total_evaluation_time_seconds": "587.0024431729689",
"transformers_version": "5.11.0",
"upper_git_hash": null,
"versions": {
"mmlu": "2",
"mmlu_abstract_algebra": 1.0,
"mmlu_anatomy": 1.0,
"mmlu_astronomy": 1.0,
"mmlu_business_ethics": 1.0,
"mmlu_clinical_knowledge": 1.0,
"mmlu_college_biology": 1.0,
"mmlu_college_chemistry": 1.0,
"mmlu_college_computer_science": 1.0,
"mmlu_college_mathematics": 1.0,
"mmlu_college_medicine": 1.0,
"mmlu_college_physics": 1.0,
"mmlu_computer_security": 1.0,
"mmlu_conceptual_physics": 1.0,
"mmlu_econometrics": 1.0,
"mmlu_electrical_engineering": 1.0,
"mmlu_elementary_mathematics": 1.0,
"mmlu_formal_logic": 1.0,
"mmlu_global_facts": 1.0,
"mmlu_high_school_biology": 1.0,
"mmlu_high_school_chemistry": 1.0,
"mmlu_high_school_computer_science": 1.0,
"mmlu_high_school_european_history": 1.0,
"mmlu_high_school_geography": 1.0,
"mmlu_high_school_government_and_politics": 1.0,
"mmlu_high_school_macroeconomics": 1.0,
"mmlu_high_school_mathematics": 1.0,
"mmlu_high_school_microeconomics": 1.0,
"mmlu_high_school_physics": 1.0,
"mmlu_high_school_psychology": 1.0,
"mmlu_high_school_statistics": 1.0,
"mmlu_high_school_us_history": 1.0,
"mmlu_high_school_world_history": 1.0,
"mmlu_human_aging": 1.0,
"mmlu_human_sexuality": 1.0,
"mmlu_humanities": "2",
"mmlu_international_law": 1.0,
"mmlu_jurisprudence": 1.0,
"mmlu_logical_fallacies": 1.0,
"mmlu_machine_learning": 1.0,
"mmlu_management": 1.0,
"mmlu_marketing": 1.0,
"mmlu_medical_genetics": 1.0,
"mmlu_miscellaneous": 1.0,
"mmlu_moral_disputes": 1.0,
"mmlu_moral_scenarios": 1.0,
"mmlu_nutrition": 1.0,
"mmlu_other": "2",
"mmlu_philosophy": 1.0,
"mmlu_prehistory": 1.0,
"mmlu_professional_accounting": 1.0,
"mmlu_professional_law": 1.0,
"mmlu_professional_medicine": 1.0,
"mmlu_professional_psychology": 1.0,
"mmlu_public_relations": 1.0,
"mmlu_security_studies": 1.0,
"mmlu_social_sciences": "2",
"mmlu_sociology": 1.0,
"mmlu_stem": "2",
"mmlu_us_foreign_policy": 1.0,
"mmlu_virology": 1.0,
"mmlu_world_religions": 1.0
}
}