4531 lines
138 KiB
JSON
4531 lines
138 KiB
JSON
{
|
|
"chat_template": "{%- if tools %}\n {{- '<|im_start|>system\\n' }}\n {%- if messages[0].role == 'system' %}\n {{- messages[0].content + '\\n\\n' }}\n {%- endif %}\n {{- \"# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n {%- for tool in tools %}\n {{- \"\\n\" }}\n {{- tool | tojson }}\n {%- endfor %}\n {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n {%- if messages[0].role == 'system' %}\n {{- '<|im_start|>system\\n' + messages[0].content + '<|im_end|>\\n' }}\n {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n {%- if message.content is string %}\n {%- set content = message.content %}\n {%- else %}\n {%- set content = '' %}\n {%- endif %}\n {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) %}\n {{- '<|im_start|>' + message.role + '\\n' + content + '<|im_end|>' + '\\n' }}\n {%- elif message.role == \"assistant\" %}\n {{- '<|im_start|>' + message.role + '\\n' + content }}\n {%- if message.tool_calls %}\n {%- for tool_call in message.tool_calls %}\n {%- if (loop.first and content) or (not loop.first) %}\n {{- '\\n' }}\n {%- endif %}\n {%- if tool_call.function %}\n {%- set tool_call = tool_call.function %}\n {%- endif %}\n {{- '<tool_call>\\n{\"name\": \"' }}\n {{- tool_call.name }}\n {{- '\", \"arguments\": ' }}\n {%- if tool_call.arguments is string %}\n {{- tool_call.arguments }}\n {%- else %}\n {{- tool_call.arguments | tojson }}\n {%- endif %}\n {{- '}\\n</tool_call>' }}\n {%- endfor %}\n {%- endif %}\n {{- '<|im_end|>\\n' }}\n {%- elif message.role == \"tool\" %}\n {%- if loop.first or (messages[loop.index0 - 1].role != \"tool\") %}\n {{- '<|im_start|>user' }}\n {%- endif %}\n {{- '\\n<tool_response>\\n' }}\n {{- content }}\n {{- '\\n</tool_response>' }}\n {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<|im_start|>assistant\\n' }}\n{%- endif %}",
|
|
"chat_template_sha": "64f85b198065d0fba2a81f37e10ed68161ce2c19a754c7100e67e0ca2ee9c326",
|
|
"config": {
|
|
"batch_size": "32",
|
|
"batch_sizes": [],
|
|
"bootstrap_iters": 100000,
|
|
"device": "cuda:0",
|
|
"fewshot_seed": 1234,
|
|
"gen_kwargs": {},
|
|
"limit": null,
|
|
"model": "vllm",
|
|
"model_args": "pretrained=kosa-labs/kosa-4B-it-v1",
|
|
"numpy_seed": 1234,
|
|
"random_seed": 0,
|
|
"torch_seed": 1234,
|
|
"use_cache": "[redacted]"
|
|
},
|
|
"configs": {
|
|
"mmlu_abstract_algebra": {
|
|
"dataset_name": "abstract_algebra",
|
|
"dataset_path": "cais/mmlu",
|
|
"description": "The following are multiple choice questions (with answers) about abstract algebra.\n\n",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_config": {
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_indices": null,
|
|
"gen_prefix": null,
|
|
"process_docs": null,
|
|
"sampler": "first_n",
|
|
"samples": null,
|
|
"split": "dev",
|
|
"target_delimiter": " "
|
|
},
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_split": "dev",
|
|
"metadata": {
|
|
"config_source": "[redacted]",
|
|
"dtype": "bfloat16",
|
|
"gpu_memory_utilization": 0.82,
|
|
"pretrained": "[redacted]",
|
|
"tensor_parallel_size": 1,
|
|
"trust_remote_code": true,
|
|
"version": 1.0
|
|
},
|
|
"metric_list": [
|
|
{
|
|
"aggregation": "mean",
|
|
"higher_is_better": true,
|
|
"metric": "acc"
|
|
}
|
|
],
|
|
"num_fewshot": 0,
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"target_delimiter": " ",
|
|
"task": "mmlu_abstract_algebra",
|
|
"task_alias": "abstract_algebra",
|
|
"test_split": "test",
|
|
"unsafe_code": false
|
|
},
|
|
"mmlu_anatomy": {
|
|
"dataset_name": "anatomy",
|
|
"dataset_path": "cais/mmlu",
|
|
"description": "The following are multiple choice questions (with answers) about anatomy.\n\n",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_config": {
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_indices": null,
|
|
"gen_prefix": null,
|
|
"process_docs": null,
|
|
"sampler": "first_n",
|
|
"samples": null,
|
|
"split": "dev",
|
|
"target_delimiter": " "
|
|
},
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_split": "dev",
|
|
"metadata": {
|
|
"config_source": "[redacted]",
|
|
"dtype": "bfloat16",
|
|
"gpu_memory_utilization": 0.82,
|
|
"pretrained": "[redacted]",
|
|
"tensor_parallel_size": 1,
|
|
"trust_remote_code": true,
|
|
"version": 1.0
|
|
},
|
|
"metric_list": [
|
|
{
|
|
"aggregation": "mean",
|
|
"higher_is_better": true,
|
|
"metric": "acc"
|
|
}
|
|
],
|
|
"num_fewshot": 0,
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"target_delimiter": " ",
|
|
"task": "mmlu_anatomy",
|
|
"task_alias": "anatomy",
|
|
"test_split": "test",
|
|
"unsafe_code": false
|
|
},
|
|
"mmlu_astronomy": {
|
|
"dataset_name": "astronomy",
|
|
"dataset_path": "cais/mmlu",
|
|
"description": "The following are multiple choice questions (with answers) about astronomy.\n\n",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_config": {
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_indices": null,
|
|
"gen_prefix": null,
|
|
"process_docs": null,
|
|
"sampler": "first_n",
|
|
"samples": null,
|
|
"split": "dev",
|
|
"target_delimiter": " "
|
|
},
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_split": "dev",
|
|
"metadata": {
|
|
"config_source": "[redacted]",
|
|
"dtype": "bfloat16",
|
|
"gpu_memory_utilization": 0.82,
|
|
"pretrained": "[redacted]",
|
|
"tensor_parallel_size": 1,
|
|
"trust_remote_code": true,
|
|
"version": 1.0
|
|
},
|
|
"metric_list": [
|
|
{
|
|
"aggregation": "mean",
|
|
"higher_is_better": true,
|
|
"metric": "acc"
|
|
}
|
|
],
|
|
"num_fewshot": 0,
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"target_delimiter": " ",
|
|
"task": "mmlu_astronomy",
|
|
"task_alias": "astronomy",
|
|
"test_split": "test",
|
|
"unsafe_code": false
|
|
},
|
|
"mmlu_business_ethics": {
|
|
"dataset_name": "business_ethics",
|
|
"dataset_path": "cais/mmlu",
|
|
"description": "The following are multiple choice questions (with answers) about business ethics.\n\n",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_config": {
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_indices": null,
|
|
"gen_prefix": null,
|
|
"process_docs": null,
|
|
"sampler": "first_n",
|
|
"samples": null,
|
|
"split": "dev",
|
|
"target_delimiter": " "
|
|
},
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_split": "dev",
|
|
"metadata": {
|
|
"config_source": "[redacted]",
|
|
"dtype": "bfloat16",
|
|
"gpu_memory_utilization": 0.82,
|
|
"pretrained": "[redacted]",
|
|
"tensor_parallel_size": 1,
|
|
"trust_remote_code": true,
|
|
"version": 1.0
|
|
},
|
|
"metric_list": [
|
|
{
|
|
"aggregation": "mean",
|
|
"higher_is_better": true,
|
|
"metric": "acc"
|
|
}
|
|
],
|
|
"num_fewshot": 0,
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"target_delimiter": " ",
|
|
"task": "mmlu_business_ethics",
|
|
"task_alias": "business_ethics",
|
|
"test_split": "test",
|
|
"unsafe_code": false
|
|
},
|
|
"mmlu_clinical_knowledge": {
|
|
"dataset_name": "clinical_knowledge",
|
|
"dataset_path": "cais/mmlu",
|
|
"description": "The following are multiple choice questions (with answers) about clinical knowledge.\n\n",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_config": {
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_indices": null,
|
|
"gen_prefix": null,
|
|
"process_docs": null,
|
|
"sampler": "first_n",
|
|
"samples": null,
|
|
"split": "dev",
|
|
"target_delimiter": " "
|
|
},
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_split": "dev",
|
|
"metadata": {
|
|
"config_source": "[redacted]",
|
|
"dtype": "bfloat16",
|
|
"gpu_memory_utilization": 0.82,
|
|
"pretrained": "[redacted]",
|
|
"tensor_parallel_size": 1,
|
|
"trust_remote_code": true,
|
|
"version": 1.0
|
|
},
|
|
"metric_list": [
|
|
{
|
|
"aggregation": "mean",
|
|
"higher_is_better": true,
|
|
"metric": "acc"
|
|
}
|
|
],
|
|
"num_fewshot": 0,
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"target_delimiter": " ",
|
|
"task": "mmlu_clinical_knowledge",
|
|
"task_alias": "clinical_knowledge",
|
|
"test_split": "test",
|
|
"unsafe_code": false
|
|
},
|
|
"mmlu_college_biology": {
|
|
"dataset_name": "college_biology",
|
|
"dataset_path": "cais/mmlu",
|
|
"description": "The following are multiple choice questions (with answers) about college biology.\n\n",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_config": {
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_indices": null,
|
|
"gen_prefix": null,
|
|
"process_docs": null,
|
|
"sampler": "first_n",
|
|
"samples": null,
|
|
"split": "dev",
|
|
"target_delimiter": " "
|
|
},
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_split": "dev",
|
|
"metadata": {
|
|
"config_source": "[redacted]",
|
|
"dtype": "bfloat16",
|
|
"gpu_memory_utilization": 0.82,
|
|
"pretrained": "[redacted]",
|
|
"tensor_parallel_size": 1,
|
|
"trust_remote_code": true,
|
|
"version": 1.0
|
|
},
|
|
"metric_list": [
|
|
{
|
|
"aggregation": "mean",
|
|
"higher_is_better": true,
|
|
"metric": "acc"
|
|
}
|
|
],
|
|
"num_fewshot": 0,
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"target_delimiter": " ",
|
|
"task": "mmlu_college_biology",
|
|
"task_alias": "college_biology",
|
|
"test_split": "test",
|
|
"unsafe_code": false
|
|
},
|
|
"mmlu_college_chemistry": {
|
|
"dataset_name": "college_chemistry",
|
|
"dataset_path": "cais/mmlu",
|
|
"description": "The following are multiple choice questions (with answers) about college chemistry.\n\n",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_config": {
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_indices": null,
|
|
"gen_prefix": null,
|
|
"process_docs": null,
|
|
"sampler": "first_n",
|
|
"samples": null,
|
|
"split": "dev",
|
|
"target_delimiter": " "
|
|
},
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_split": "dev",
|
|
"metadata": {
|
|
"config_source": "[redacted]",
|
|
"dtype": "bfloat16",
|
|
"gpu_memory_utilization": 0.82,
|
|
"pretrained": "[redacted]",
|
|
"tensor_parallel_size": 1,
|
|
"trust_remote_code": true,
|
|
"version": 1.0
|
|
},
|
|
"metric_list": [
|
|
{
|
|
"aggregation": "mean",
|
|
"higher_is_better": true,
|
|
"metric": "acc"
|
|
}
|
|
],
|
|
"num_fewshot": 0,
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"target_delimiter": " ",
|
|
"task": "mmlu_college_chemistry",
|
|
"task_alias": "college_chemistry",
|
|
"test_split": "test",
|
|
"unsafe_code": false
|
|
},
|
|
"mmlu_college_computer_science": {
|
|
"dataset_name": "college_computer_science",
|
|
"dataset_path": "cais/mmlu",
|
|
"description": "The following are multiple choice questions (with answers) about college computer science.\n\n",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_config": {
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_indices": null,
|
|
"gen_prefix": null,
|
|
"process_docs": null,
|
|
"sampler": "first_n",
|
|
"samples": null,
|
|
"split": "dev",
|
|
"target_delimiter": " "
|
|
},
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_split": "dev",
|
|
"metadata": {
|
|
"config_source": "[redacted]",
|
|
"dtype": "bfloat16",
|
|
"gpu_memory_utilization": 0.82,
|
|
"pretrained": "[redacted]",
|
|
"tensor_parallel_size": 1,
|
|
"trust_remote_code": true,
|
|
"version": 1.0
|
|
},
|
|
"metric_list": [
|
|
{
|
|
"aggregation": "mean",
|
|
"higher_is_better": true,
|
|
"metric": "acc"
|
|
}
|
|
],
|
|
"num_fewshot": 0,
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"target_delimiter": " ",
|
|
"task": "mmlu_college_computer_science",
|
|
"task_alias": "college_computer_science",
|
|
"test_split": "test",
|
|
"unsafe_code": false
|
|
},
|
|
"mmlu_college_mathematics": {
|
|
"dataset_name": "college_mathematics",
|
|
"dataset_path": "cais/mmlu",
|
|
"description": "The following are multiple choice questions (with answers) about college mathematics.\n\n",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_config": {
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_indices": null,
|
|
"gen_prefix": null,
|
|
"process_docs": null,
|
|
"sampler": "first_n",
|
|
"samples": null,
|
|
"split": "dev",
|
|
"target_delimiter": " "
|
|
},
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_split": "dev",
|
|
"metadata": {
|
|
"config_source": "[redacted]",
|
|
"dtype": "bfloat16",
|
|
"gpu_memory_utilization": 0.82,
|
|
"pretrained": "[redacted]",
|
|
"tensor_parallel_size": 1,
|
|
"trust_remote_code": true,
|
|
"version": 1.0
|
|
},
|
|
"metric_list": [
|
|
{
|
|
"aggregation": "mean",
|
|
"higher_is_better": true,
|
|
"metric": "acc"
|
|
}
|
|
],
|
|
"num_fewshot": 0,
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"target_delimiter": " ",
|
|
"task": "mmlu_college_mathematics",
|
|
"task_alias": "college_mathematics",
|
|
"test_split": "test",
|
|
"unsafe_code": false
|
|
},
|
|
"mmlu_college_medicine": {
|
|
"dataset_name": "college_medicine",
|
|
"dataset_path": "cais/mmlu",
|
|
"description": "The following are multiple choice questions (with answers) about college medicine.\n\n",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_config": {
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_indices": null,
|
|
"gen_prefix": null,
|
|
"process_docs": null,
|
|
"sampler": "first_n",
|
|
"samples": null,
|
|
"split": "dev",
|
|
"target_delimiter": " "
|
|
},
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_split": "dev",
|
|
"metadata": {
|
|
"config_source": "[redacted]",
|
|
"dtype": "bfloat16",
|
|
"gpu_memory_utilization": 0.82,
|
|
"pretrained": "[redacted]",
|
|
"tensor_parallel_size": 1,
|
|
"trust_remote_code": true,
|
|
"version": 1.0
|
|
},
|
|
"metric_list": [
|
|
{
|
|
"aggregation": "mean",
|
|
"higher_is_better": true,
|
|
"metric": "acc"
|
|
}
|
|
],
|
|
"num_fewshot": 0,
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"target_delimiter": " ",
|
|
"task": "mmlu_college_medicine",
|
|
"task_alias": "college_medicine",
|
|
"test_split": "test",
|
|
"unsafe_code": false
|
|
},
|
|
"mmlu_college_physics": {
|
|
"dataset_name": "college_physics",
|
|
"dataset_path": "cais/mmlu",
|
|
"description": "The following are multiple choice questions (with answers) about college physics.\n\n",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_config": {
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_indices": null,
|
|
"gen_prefix": null,
|
|
"process_docs": null,
|
|
"sampler": "first_n",
|
|
"samples": null,
|
|
"split": "dev",
|
|
"target_delimiter": " "
|
|
},
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_split": "dev",
|
|
"metadata": {
|
|
"config_source": "[redacted]",
|
|
"dtype": "bfloat16",
|
|
"gpu_memory_utilization": 0.82,
|
|
"pretrained": "[redacted]",
|
|
"tensor_parallel_size": 1,
|
|
"trust_remote_code": true,
|
|
"version": 1.0
|
|
},
|
|
"metric_list": [
|
|
{
|
|
"aggregation": "mean",
|
|
"higher_is_better": true,
|
|
"metric": "acc"
|
|
}
|
|
],
|
|
"num_fewshot": 0,
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"target_delimiter": " ",
|
|
"task": "mmlu_college_physics",
|
|
"task_alias": "college_physics",
|
|
"test_split": "test",
|
|
"unsafe_code": false
|
|
},
|
|
"mmlu_computer_security": {
|
|
"dataset_name": "computer_security",
|
|
"dataset_path": "cais/mmlu",
|
|
"description": "The following are multiple choice questions (with answers) about computer security.\n\n",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_config": {
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_indices": null,
|
|
"gen_prefix": null,
|
|
"process_docs": null,
|
|
"sampler": "first_n",
|
|
"samples": null,
|
|
"split": "dev",
|
|
"target_delimiter": " "
|
|
},
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_split": "dev",
|
|
"metadata": {
|
|
"config_source": "[redacted]",
|
|
"dtype": "bfloat16",
|
|
"gpu_memory_utilization": 0.82,
|
|
"pretrained": "[redacted]",
|
|
"tensor_parallel_size": 1,
|
|
"trust_remote_code": true,
|
|
"version": 1.0
|
|
},
|
|
"metric_list": [
|
|
{
|
|
"aggregation": "mean",
|
|
"higher_is_better": true,
|
|
"metric": "acc"
|
|
}
|
|
],
|
|
"num_fewshot": 0,
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"target_delimiter": " ",
|
|
"task": "mmlu_computer_security",
|
|
"task_alias": "computer_security",
|
|
"test_split": "test",
|
|
"unsafe_code": false
|
|
},
|
|
"mmlu_conceptual_physics": {
|
|
"dataset_name": "conceptual_physics",
|
|
"dataset_path": "cais/mmlu",
|
|
"description": "The following are multiple choice questions (with answers) about conceptual physics.\n\n",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_config": {
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_indices": null,
|
|
"gen_prefix": null,
|
|
"process_docs": null,
|
|
"sampler": "first_n",
|
|
"samples": null,
|
|
"split": "dev",
|
|
"target_delimiter": " "
|
|
},
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_split": "dev",
|
|
"metadata": {
|
|
"config_source": "[redacted]",
|
|
"dtype": "bfloat16",
|
|
"gpu_memory_utilization": 0.82,
|
|
"pretrained": "[redacted]",
|
|
"tensor_parallel_size": 1,
|
|
"trust_remote_code": true,
|
|
"version": 1.0
|
|
},
|
|
"metric_list": [
|
|
{
|
|
"aggregation": "mean",
|
|
"higher_is_better": true,
|
|
"metric": "acc"
|
|
}
|
|
],
|
|
"num_fewshot": 0,
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"target_delimiter": " ",
|
|
"task": "mmlu_conceptual_physics",
|
|
"task_alias": "conceptual_physics",
|
|
"test_split": "test",
|
|
"unsafe_code": false
|
|
},
|
|
"mmlu_econometrics": {
|
|
"dataset_name": "econometrics",
|
|
"dataset_path": "cais/mmlu",
|
|
"description": "The following are multiple choice questions (with answers) about econometrics.\n\n",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_config": {
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_indices": null,
|
|
"gen_prefix": null,
|
|
"process_docs": null,
|
|
"sampler": "first_n",
|
|
"samples": null,
|
|
"split": "dev",
|
|
"target_delimiter": " "
|
|
},
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_split": "dev",
|
|
"metadata": {
|
|
"config_source": "[redacted]",
|
|
"dtype": "bfloat16",
|
|
"gpu_memory_utilization": 0.82,
|
|
"pretrained": "[redacted]",
|
|
"tensor_parallel_size": 1,
|
|
"trust_remote_code": true,
|
|
"version": 1.0
|
|
},
|
|
"metric_list": [
|
|
{
|
|
"aggregation": "mean",
|
|
"higher_is_better": true,
|
|
"metric": "acc"
|
|
}
|
|
],
|
|
"num_fewshot": 0,
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"target_delimiter": " ",
|
|
"task": "mmlu_econometrics",
|
|
"task_alias": "econometrics",
|
|
"test_split": "test",
|
|
"unsafe_code": false
|
|
},
|
|
"mmlu_electrical_engineering": {
|
|
"dataset_name": "electrical_engineering",
|
|
"dataset_path": "cais/mmlu",
|
|
"description": "The following are multiple choice questions (with answers) about electrical engineering.\n\n",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_config": {
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_indices": null,
|
|
"gen_prefix": null,
|
|
"process_docs": null,
|
|
"sampler": "first_n",
|
|
"samples": null,
|
|
"split": "dev",
|
|
"target_delimiter": " "
|
|
},
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_split": "dev",
|
|
"metadata": {
|
|
"config_source": "[redacted]",
|
|
"dtype": "bfloat16",
|
|
"gpu_memory_utilization": 0.82,
|
|
"pretrained": "[redacted]",
|
|
"tensor_parallel_size": 1,
|
|
"trust_remote_code": true,
|
|
"version": 1.0
|
|
},
|
|
"metric_list": [
|
|
{
|
|
"aggregation": "mean",
|
|
"higher_is_better": true,
|
|
"metric": "acc"
|
|
}
|
|
],
|
|
"num_fewshot": 0,
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"target_delimiter": " ",
|
|
"task": "mmlu_electrical_engineering",
|
|
"task_alias": "electrical_engineering",
|
|
"test_split": "test",
|
|
"unsafe_code": false
|
|
},
|
|
"mmlu_elementary_mathematics": {
|
|
"dataset_name": "elementary_mathematics",
|
|
"dataset_path": "cais/mmlu",
|
|
"description": "The following are multiple choice questions (with answers) about elementary mathematics.\n\n",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_config": {
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_indices": null,
|
|
"gen_prefix": null,
|
|
"process_docs": null,
|
|
"sampler": "first_n",
|
|
"samples": null,
|
|
"split": "dev",
|
|
"target_delimiter": " "
|
|
},
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_split": "dev",
|
|
"metadata": {
|
|
"config_source": "[redacted]",
|
|
"dtype": "bfloat16",
|
|
"gpu_memory_utilization": 0.82,
|
|
"pretrained": "[redacted]",
|
|
"tensor_parallel_size": 1,
|
|
"trust_remote_code": true,
|
|
"version": 1.0
|
|
},
|
|
"metric_list": [
|
|
{
|
|
"aggregation": "mean",
|
|
"higher_is_better": true,
|
|
"metric": "acc"
|
|
}
|
|
],
|
|
"num_fewshot": 0,
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"target_delimiter": " ",
|
|
"task": "mmlu_elementary_mathematics",
|
|
"task_alias": "elementary_mathematics",
|
|
"test_split": "test",
|
|
"unsafe_code": false
|
|
},
|
|
"mmlu_formal_logic": {
|
|
"dataset_name": "formal_logic",
|
|
"dataset_path": "cais/mmlu",
|
|
"description": "The following are multiple choice questions (with answers) about formal logic.\n\n",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_config": {
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_indices": null,
|
|
"gen_prefix": null,
|
|
"process_docs": null,
|
|
"sampler": "first_n",
|
|
"samples": null,
|
|
"split": "dev",
|
|
"target_delimiter": " "
|
|
},
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_split": "dev",
|
|
"metadata": {
|
|
"config_source": "[redacted]",
|
|
"dtype": "bfloat16",
|
|
"gpu_memory_utilization": 0.82,
|
|
"pretrained": "[redacted]",
|
|
"tensor_parallel_size": 1,
|
|
"trust_remote_code": true,
|
|
"version": 1.0
|
|
},
|
|
"metric_list": [
|
|
{
|
|
"aggregation": "mean",
|
|
"higher_is_better": true,
|
|
"metric": "acc"
|
|
}
|
|
],
|
|
"num_fewshot": 0,
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"target_delimiter": " ",
|
|
"task": "mmlu_formal_logic",
|
|
"task_alias": "formal_logic",
|
|
"test_split": "test",
|
|
"unsafe_code": false
|
|
},
|
|
"mmlu_global_facts": {
|
|
"dataset_name": "global_facts",
|
|
"dataset_path": "cais/mmlu",
|
|
"description": "The following are multiple choice questions (with answers) about global facts.\n\n",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_config": {
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_indices": null,
|
|
"gen_prefix": null,
|
|
"process_docs": null,
|
|
"sampler": "first_n",
|
|
"samples": null,
|
|
"split": "dev",
|
|
"target_delimiter": " "
|
|
},
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_split": "dev",
|
|
"metadata": {
|
|
"config_source": "[redacted]",
|
|
"dtype": "bfloat16",
|
|
"gpu_memory_utilization": 0.82,
|
|
"pretrained": "[redacted]",
|
|
"tensor_parallel_size": 1,
|
|
"trust_remote_code": true,
|
|
"version": 1.0
|
|
},
|
|
"metric_list": [
|
|
{
|
|
"aggregation": "mean",
|
|
"higher_is_better": true,
|
|
"metric": "acc"
|
|
}
|
|
],
|
|
"num_fewshot": 0,
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"target_delimiter": " ",
|
|
"task": "mmlu_global_facts",
|
|
"task_alias": "global_facts",
|
|
"test_split": "test",
|
|
"unsafe_code": false
|
|
},
|
|
"mmlu_high_school_biology": {
|
|
"dataset_name": "high_school_biology",
|
|
"dataset_path": "cais/mmlu",
|
|
"description": "The following are multiple choice questions (with answers) about high school biology.\n\n",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_config": {
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_indices": null,
|
|
"gen_prefix": null,
|
|
"process_docs": null,
|
|
"sampler": "first_n",
|
|
"samples": null,
|
|
"split": "dev",
|
|
"target_delimiter": " "
|
|
},
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_split": "dev",
|
|
"metadata": {
|
|
"config_source": "[redacted]",
|
|
"dtype": "bfloat16",
|
|
"gpu_memory_utilization": 0.82,
|
|
"pretrained": "[redacted]",
|
|
"tensor_parallel_size": 1,
|
|
"trust_remote_code": true,
|
|
"version": 1.0
|
|
},
|
|
"metric_list": [
|
|
{
|
|
"aggregation": "mean",
|
|
"higher_is_better": true,
|
|
"metric": "acc"
|
|
}
|
|
],
|
|
"num_fewshot": 0,
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"target_delimiter": " ",
|
|
"task": "mmlu_high_school_biology",
|
|
"task_alias": "high_school_biology",
|
|
"test_split": "test",
|
|
"unsafe_code": false
|
|
},
|
|
"mmlu_high_school_chemistry": {
|
|
"dataset_name": "high_school_chemistry",
|
|
"dataset_path": "cais/mmlu",
|
|
"description": "The following are multiple choice questions (with answers) about high school chemistry.\n\n",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_config": {
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_indices": null,
|
|
"gen_prefix": null,
|
|
"process_docs": null,
|
|
"sampler": "first_n",
|
|
"samples": null,
|
|
"split": "dev",
|
|
"target_delimiter": " "
|
|
},
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_split": "dev",
|
|
"metadata": {
|
|
"config_source": "[redacted]",
|
|
"dtype": "bfloat16",
|
|
"gpu_memory_utilization": 0.82,
|
|
"pretrained": "[redacted]",
|
|
"tensor_parallel_size": 1,
|
|
"trust_remote_code": true,
|
|
"version": 1.0
|
|
},
|
|
"metric_list": [
|
|
{
|
|
"aggregation": "mean",
|
|
"higher_is_better": true,
|
|
"metric": "acc"
|
|
}
|
|
],
|
|
"num_fewshot": 0,
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"target_delimiter": " ",
|
|
"task": "mmlu_high_school_chemistry",
|
|
"task_alias": "high_school_chemistry",
|
|
"test_split": "test",
|
|
"unsafe_code": false
|
|
},
|
|
"mmlu_high_school_computer_science": {
|
|
"dataset_name": "high_school_computer_science",
|
|
"dataset_path": "cais/mmlu",
|
|
"description": "The following are multiple choice questions (with answers) about high school computer science.\n\n",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_config": {
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_indices": null,
|
|
"gen_prefix": null,
|
|
"process_docs": null,
|
|
"sampler": "first_n",
|
|
"samples": null,
|
|
"split": "dev",
|
|
"target_delimiter": " "
|
|
},
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_split": "dev",
|
|
"metadata": {
|
|
"config_source": "[redacted]",
|
|
"dtype": "bfloat16",
|
|
"gpu_memory_utilization": 0.82,
|
|
"pretrained": "[redacted]",
|
|
"tensor_parallel_size": 1,
|
|
"trust_remote_code": true,
|
|
"version": 1.0
|
|
},
|
|
"metric_list": [
|
|
{
|
|
"aggregation": "mean",
|
|
"higher_is_better": true,
|
|
"metric": "acc"
|
|
}
|
|
],
|
|
"num_fewshot": 0,
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"target_delimiter": " ",
|
|
"task": "mmlu_high_school_computer_science",
|
|
"task_alias": "high_school_computer_science",
|
|
"test_split": "test",
|
|
"unsafe_code": false
|
|
},
|
|
"mmlu_high_school_european_history": {
|
|
"dataset_name": "high_school_european_history",
|
|
"dataset_path": "cais/mmlu",
|
|
"description": "The following are multiple choice questions (with answers) about high school european history.\n\n",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_config": {
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_indices": null,
|
|
"gen_prefix": null,
|
|
"process_docs": null,
|
|
"sampler": "first_n",
|
|
"samples": null,
|
|
"split": "dev",
|
|
"target_delimiter": " "
|
|
},
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_split": "dev",
|
|
"metadata": {
|
|
"config_source": "[redacted]",
|
|
"dtype": "bfloat16",
|
|
"gpu_memory_utilization": 0.82,
|
|
"pretrained": "[redacted]",
|
|
"tensor_parallel_size": 1,
|
|
"trust_remote_code": true,
|
|
"version": 1.0
|
|
},
|
|
"metric_list": [
|
|
{
|
|
"aggregation": "mean",
|
|
"higher_is_better": true,
|
|
"metric": "acc"
|
|
}
|
|
],
|
|
"num_fewshot": 0,
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"target_delimiter": " ",
|
|
"task": "mmlu_high_school_european_history",
|
|
"task_alias": "high_school_european_history",
|
|
"test_split": "test",
|
|
"unsafe_code": false
|
|
},
|
|
"mmlu_high_school_geography": {
|
|
"dataset_name": "high_school_geography",
|
|
"dataset_path": "cais/mmlu",
|
|
"description": "The following are multiple choice questions (with answers) about high school geography.\n\n",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_config": {
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_indices": null,
|
|
"gen_prefix": null,
|
|
"process_docs": null,
|
|
"sampler": "first_n",
|
|
"samples": null,
|
|
"split": "dev",
|
|
"target_delimiter": " "
|
|
},
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_split": "dev",
|
|
"metadata": {
|
|
"config_source": "[redacted]",
|
|
"dtype": "bfloat16",
|
|
"gpu_memory_utilization": 0.82,
|
|
"pretrained": "[redacted]",
|
|
"tensor_parallel_size": 1,
|
|
"trust_remote_code": true,
|
|
"version": 1.0
|
|
},
|
|
"metric_list": [
|
|
{
|
|
"aggregation": "mean",
|
|
"higher_is_better": true,
|
|
"metric": "acc"
|
|
}
|
|
],
|
|
"num_fewshot": 0,
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"target_delimiter": " ",
|
|
"task": "mmlu_high_school_geography",
|
|
"task_alias": "high_school_geography",
|
|
"test_split": "test",
|
|
"unsafe_code": false
|
|
},
|
|
"mmlu_high_school_government_and_politics": {
|
|
"dataset_name": "high_school_government_and_politics",
|
|
"dataset_path": "cais/mmlu",
|
|
"description": "The following are multiple choice questions (with answers) about high school government and politics.\n\n",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_config": {
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_indices": null,
|
|
"gen_prefix": null,
|
|
"process_docs": null,
|
|
"sampler": "first_n",
|
|
"samples": null,
|
|
"split": "dev",
|
|
"target_delimiter": " "
|
|
},
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_split": "dev",
|
|
"metadata": {
|
|
"config_source": "[redacted]",
|
|
"dtype": "bfloat16",
|
|
"gpu_memory_utilization": 0.82,
|
|
"pretrained": "[redacted]",
|
|
"tensor_parallel_size": 1,
|
|
"trust_remote_code": true,
|
|
"version": 1.0
|
|
},
|
|
"metric_list": [
|
|
{
|
|
"aggregation": "mean",
|
|
"higher_is_better": true,
|
|
"metric": "acc"
|
|
}
|
|
],
|
|
"num_fewshot": 0,
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"target_delimiter": " ",
|
|
"task": "mmlu_high_school_government_and_politics",
|
|
"task_alias": "high_school_government_and_politics",
|
|
"test_split": "test",
|
|
"unsafe_code": false
|
|
},
|
|
"mmlu_high_school_macroeconomics": {
|
|
"dataset_name": "high_school_macroeconomics",
|
|
"dataset_path": "cais/mmlu",
|
|
"description": "The following are multiple choice questions (with answers) about high school macroeconomics.\n\n",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_config": {
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_indices": null,
|
|
"gen_prefix": null,
|
|
"process_docs": null,
|
|
"sampler": "first_n",
|
|
"samples": null,
|
|
"split": "dev",
|
|
"target_delimiter": " "
|
|
},
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_split": "dev",
|
|
"metadata": {
|
|
"config_source": "[redacted]",
|
|
"dtype": "bfloat16",
|
|
"gpu_memory_utilization": 0.82,
|
|
"pretrained": "[redacted]",
|
|
"tensor_parallel_size": 1,
|
|
"trust_remote_code": true,
|
|
"version": 1.0
|
|
},
|
|
"metric_list": [
|
|
{
|
|
"aggregation": "mean",
|
|
"higher_is_better": true,
|
|
"metric": "acc"
|
|
}
|
|
],
|
|
"num_fewshot": 0,
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"target_delimiter": " ",
|
|
"task": "mmlu_high_school_macroeconomics",
|
|
"task_alias": "high_school_macroeconomics",
|
|
"test_split": "test",
|
|
"unsafe_code": false
|
|
},
|
|
"mmlu_high_school_mathematics": {
|
|
"dataset_name": "high_school_mathematics",
|
|
"dataset_path": "cais/mmlu",
|
|
"description": "The following are multiple choice questions (with answers) about high school mathematics.\n\n",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_config": {
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_indices": null,
|
|
"gen_prefix": null,
|
|
"process_docs": null,
|
|
"sampler": "first_n",
|
|
"samples": null,
|
|
"split": "dev",
|
|
"target_delimiter": " "
|
|
},
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_split": "dev",
|
|
"metadata": {
|
|
"config_source": "[redacted]",
|
|
"dtype": "bfloat16",
|
|
"gpu_memory_utilization": 0.82,
|
|
"pretrained": "[redacted]",
|
|
"tensor_parallel_size": 1,
|
|
"trust_remote_code": true,
|
|
"version": 1.0
|
|
},
|
|
"metric_list": [
|
|
{
|
|
"aggregation": "mean",
|
|
"higher_is_better": true,
|
|
"metric": "acc"
|
|
}
|
|
],
|
|
"num_fewshot": 0,
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"target_delimiter": " ",
|
|
"task": "mmlu_high_school_mathematics",
|
|
"task_alias": "high_school_mathematics",
|
|
"test_split": "test",
|
|
"unsafe_code": false
|
|
},
|
|
"mmlu_high_school_microeconomics": {
|
|
"dataset_name": "high_school_microeconomics",
|
|
"dataset_path": "cais/mmlu",
|
|
"description": "The following are multiple choice questions (with answers) about high school microeconomics.\n\n",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_config": {
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_indices": null,
|
|
"gen_prefix": null,
|
|
"process_docs": null,
|
|
"sampler": "first_n",
|
|
"samples": null,
|
|
"split": "dev",
|
|
"target_delimiter": " "
|
|
},
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_split": "dev",
|
|
"metadata": {
|
|
"config_source": "[redacted]",
|
|
"dtype": "bfloat16",
|
|
"gpu_memory_utilization": 0.82,
|
|
"pretrained": "[redacted]",
|
|
"tensor_parallel_size": 1,
|
|
"trust_remote_code": true,
|
|
"version": 1.0
|
|
},
|
|
"metric_list": [
|
|
{
|
|
"aggregation": "mean",
|
|
"higher_is_better": true,
|
|
"metric": "acc"
|
|
}
|
|
],
|
|
"num_fewshot": 0,
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"target_delimiter": " ",
|
|
"task": "mmlu_high_school_microeconomics",
|
|
"task_alias": "high_school_microeconomics",
|
|
"test_split": "test",
|
|
"unsafe_code": false
|
|
},
|
|
"mmlu_high_school_physics": {
|
|
"dataset_name": "high_school_physics",
|
|
"dataset_path": "cais/mmlu",
|
|
"description": "The following are multiple choice questions (with answers) about high school physics.\n\n",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_config": {
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_indices": null,
|
|
"gen_prefix": null,
|
|
"process_docs": null,
|
|
"sampler": "first_n",
|
|
"samples": null,
|
|
"split": "dev",
|
|
"target_delimiter": " "
|
|
},
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_split": "dev",
|
|
"metadata": {
|
|
"config_source": "[redacted]",
|
|
"dtype": "bfloat16",
|
|
"gpu_memory_utilization": 0.82,
|
|
"pretrained": "[redacted]",
|
|
"tensor_parallel_size": 1,
|
|
"trust_remote_code": true,
|
|
"version": 1.0
|
|
},
|
|
"metric_list": [
|
|
{
|
|
"aggregation": "mean",
|
|
"higher_is_better": true,
|
|
"metric": "acc"
|
|
}
|
|
],
|
|
"num_fewshot": 0,
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"target_delimiter": " ",
|
|
"task": "mmlu_high_school_physics",
|
|
"task_alias": "high_school_physics",
|
|
"test_split": "test",
|
|
"unsafe_code": false
|
|
},
|
|
"mmlu_high_school_psychology": {
|
|
"dataset_name": "high_school_psychology",
|
|
"dataset_path": "cais/mmlu",
|
|
"description": "The following are multiple choice questions (with answers) about high school psychology.\n\n",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_config": {
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_indices": null,
|
|
"gen_prefix": null,
|
|
"process_docs": null,
|
|
"sampler": "first_n",
|
|
"samples": null,
|
|
"split": "dev",
|
|
"target_delimiter": " "
|
|
},
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_split": "dev",
|
|
"metadata": {
|
|
"config_source": "[redacted]",
|
|
"dtype": "bfloat16",
|
|
"gpu_memory_utilization": 0.82,
|
|
"pretrained": "[redacted]",
|
|
"tensor_parallel_size": 1,
|
|
"trust_remote_code": true,
|
|
"version": 1.0
|
|
},
|
|
"metric_list": [
|
|
{
|
|
"aggregation": "mean",
|
|
"higher_is_better": true,
|
|
"metric": "acc"
|
|
}
|
|
],
|
|
"num_fewshot": 0,
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"target_delimiter": " ",
|
|
"task": "mmlu_high_school_psychology",
|
|
"task_alias": "high_school_psychology",
|
|
"test_split": "test",
|
|
"unsafe_code": false
|
|
},
|
|
"mmlu_high_school_statistics": {
|
|
"dataset_name": "high_school_statistics",
|
|
"dataset_path": "cais/mmlu",
|
|
"description": "The following are multiple choice questions (with answers) about high school statistics.\n\n",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_config": {
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_indices": null,
|
|
"gen_prefix": null,
|
|
"process_docs": null,
|
|
"sampler": "first_n",
|
|
"samples": null,
|
|
"split": "dev",
|
|
"target_delimiter": " "
|
|
},
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_split": "dev",
|
|
"metadata": {
|
|
"config_source": "[redacted]",
|
|
"dtype": "bfloat16",
|
|
"gpu_memory_utilization": 0.82,
|
|
"pretrained": "[redacted]",
|
|
"tensor_parallel_size": 1,
|
|
"trust_remote_code": true,
|
|
"version": 1.0
|
|
},
|
|
"metric_list": [
|
|
{
|
|
"aggregation": "mean",
|
|
"higher_is_better": true,
|
|
"metric": "acc"
|
|
}
|
|
],
|
|
"num_fewshot": 0,
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"target_delimiter": " ",
|
|
"task": "mmlu_high_school_statistics",
|
|
"task_alias": "high_school_statistics",
|
|
"test_split": "test",
|
|
"unsafe_code": false
|
|
},
|
|
"mmlu_high_school_us_history": {
|
|
"dataset_name": "high_school_us_history",
|
|
"dataset_path": "cais/mmlu",
|
|
"description": "The following are multiple choice questions (with answers) about high school us history.\n\n",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_config": {
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_indices": null,
|
|
"gen_prefix": null,
|
|
"process_docs": null,
|
|
"sampler": "first_n",
|
|
"samples": null,
|
|
"split": "dev",
|
|
"target_delimiter": " "
|
|
},
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_split": "dev",
|
|
"metadata": {
|
|
"config_source": "[redacted]",
|
|
"dtype": "bfloat16",
|
|
"gpu_memory_utilization": 0.82,
|
|
"pretrained": "[redacted]",
|
|
"tensor_parallel_size": 1,
|
|
"trust_remote_code": true,
|
|
"version": 1.0
|
|
},
|
|
"metric_list": [
|
|
{
|
|
"aggregation": "mean",
|
|
"higher_is_better": true,
|
|
"metric": "acc"
|
|
}
|
|
],
|
|
"num_fewshot": 0,
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"target_delimiter": " ",
|
|
"task": "mmlu_high_school_us_history",
|
|
"task_alias": "high_school_us_history",
|
|
"test_split": "test",
|
|
"unsafe_code": false
|
|
},
|
|
"mmlu_high_school_world_history": {
|
|
"dataset_name": "high_school_world_history",
|
|
"dataset_path": "cais/mmlu",
|
|
"description": "The following are multiple choice questions (with answers) about high school world history.\n\n",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_config": {
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_indices": null,
|
|
"gen_prefix": null,
|
|
"process_docs": null,
|
|
"sampler": "first_n",
|
|
"samples": null,
|
|
"split": "dev",
|
|
"target_delimiter": " "
|
|
},
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_split": "dev",
|
|
"metadata": {
|
|
"config_source": "[redacted]",
|
|
"dtype": "bfloat16",
|
|
"gpu_memory_utilization": 0.82,
|
|
"pretrained": "[redacted]",
|
|
"tensor_parallel_size": 1,
|
|
"trust_remote_code": true,
|
|
"version": 1.0
|
|
},
|
|
"metric_list": [
|
|
{
|
|
"aggregation": "mean",
|
|
"higher_is_better": true,
|
|
"metric": "acc"
|
|
}
|
|
],
|
|
"num_fewshot": 0,
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"target_delimiter": " ",
|
|
"task": "mmlu_high_school_world_history",
|
|
"task_alias": "high_school_world_history",
|
|
"test_split": "test",
|
|
"unsafe_code": false
|
|
},
|
|
"mmlu_human_aging": {
|
|
"dataset_name": "human_aging",
|
|
"dataset_path": "cais/mmlu",
|
|
"description": "The following are multiple choice questions (with answers) about human aging.\n\n",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_config": {
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_indices": null,
|
|
"gen_prefix": null,
|
|
"process_docs": null,
|
|
"sampler": "first_n",
|
|
"samples": null,
|
|
"split": "dev",
|
|
"target_delimiter": " "
|
|
},
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_split": "dev",
|
|
"metadata": {
|
|
"config_source": "[redacted]",
|
|
"dtype": "bfloat16",
|
|
"gpu_memory_utilization": 0.82,
|
|
"pretrained": "[redacted]",
|
|
"tensor_parallel_size": 1,
|
|
"trust_remote_code": true,
|
|
"version": 1.0
|
|
},
|
|
"metric_list": [
|
|
{
|
|
"aggregation": "mean",
|
|
"higher_is_better": true,
|
|
"metric": "acc"
|
|
}
|
|
],
|
|
"num_fewshot": 0,
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"target_delimiter": " ",
|
|
"task": "mmlu_human_aging",
|
|
"task_alias": "human_aging",
|
|
"test_split": "test",
|
|
"unsafe_code": false
|
|
},
|
|
"mmlu_human_sexuality": {
|
|
"dataset_name": "human_sexuality",
|
|
"dataset_path": "cais/mmlu",
|
|
"description": "The following are multiple choice questions (with answers) about human sexuality.\n\n",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_config": {
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_indices": null,
|
|
"gen_prefix": null,
|
|
"process_docs": null,
|
|
"sampler": "first_n",
|
|
"samples": null,
|
|
"split": "dev",
|
|
"target_delimiter": " "
|
|
},
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_split": "dev",
|
|
"metadata": {
|
|
"config_source": "[redacted]",
|
|
"dtype": "bfloat16",
|
|
"gpu_memory_utilization": 0.82,
|
|
"pretrained": "[redacted]",
|
|
"tensor_parallel_size": 1,
|
|
"trust_remote_code": true,
|
|
"version": 1.0
|
|
},
|
|
"metric_list": [
|
|
{
|
|
"aggregation": "mean",
|
|
"higher_is_better": true,
|
|
"metric": "acc"
|
|
}
|
|
],
|
|
"num_fewshot": 0,
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"target_delimiter": " ",
|
|
"task": "mmlu_human_sexuality",
|
|
"task_alias": "human_sexuality",
|
|
"test_split": "test",
|
|
"unsafe_code": false
|
|
},
|
|
"mmlu_international_law": {
|
|
"dataset_name": "international_law",
|
|
"dataset_path": "cais/mmlu",
|
|
"description": "The following are multiple choice questions (with answers) about international law.\n\n",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_config": {
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_indices": null,
|
|
"gen_prefix": null,
|
|
"process_docs": null,
|
|
"sampler": "first_n",
|
|
"samples": null,
|
|
"split": "dev",
|
|
"target_delimiter": " "
|
|
},
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_split": "dev",
|
|
"metadata": {
|
|
"config_source": "[redacted]",
|
|
"dtype": "bfloat16",
|
|
"gpu_memory_utilization": 0.82,
|
|
"pretrained": "[redacted]",
|
|
"tensor_parallel_size": 1,
|
|
"trust_remote_code": true,
|
|
"version": 1.0
|
|
},
|
|
"metric_list": [
|
|
{
|
|
"aggregation": "mean",
|
|
"higher_is_better": true,
|
|
"metric": "acc"
|
|
}
|
|
],
|
|
"num_fewshot": 0,
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"target_delimiter": " ",
|
|
"task": "mmlu_international_law",
|
|
"task_alias": "international_law",
|
|
"test_split": "test",
|
|
"unsafe_code": false
|
|
},
|
|
"mmlu_jurisprudence": {
|
|
"dataset_name": "jurisprudence",
|
|
"dataset_path": "cais/mmlu",
|
|
"description": "The following are multiple choice questions (with answers) about jurisprudence.\n\n",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_config": {
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_indices": null,
|
|
"gen_prefix": null,
|
|
"process_docs": null,
|
|
"sampler": "first_n",
|
|
"samples": null,
|
|
"split": "dev",
|
|
"target_delimiter": " "
|
|
},
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_split": "dev",
|
|
"metadata": {
|
|
"config_source": "[redacted]",
|
|
"dtype": "bfloat16",
|
|
"gpu_memory_utilization": 0.82,
|
|
"pretrained": "[redacted]",
|
|
"tensor_parallel_size": 1,
|
|
"trust_remote_code": true,
|
|
"version": 1.0
|
|
},
|
|
"metric_list": [
|
|
{
|
|
"aggregation": "mean",
|
|
"higher_is_better": true,
|
|
"metric": "acc"
|
|
}
|
|
],
|
|
"num_fewshot": 0,
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"target_delimiter": " ",
|
|
"task": "mmlu_jurisprudence",
|
|
"task_alias": "jurisprudence",
|
|
"test_split": "test",
|
|
"unsafe_code": false
|
|
},
|
|
"mmlu_logical_fallacies": {
|
|
"dataset_name": "logical_fallacies",
|
|
"dataset_path": "cais/mmlu",
|
|
"description": "The following are multiple choice questions (with answers) about logical fallacies.\n\n",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_config": {
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_indices": null,
|
|
"gen_prefix": null,
|
|
"process_docs": null,
|
|
"sampler": "first_n",
|
|
"samples": null,
|
|
"split": "dev",
|
|
"target_delimiter": " "
|
|
},
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_split": "dev",
|
|
"metadata": {
|
|
"config_source": "[redacted]",
|
|
"dtype": "bfloat16",
|
|
"gpu_memory_utilization": 0.82,
|
|
"pretrained": "[redacted]",
|
|
"tensor_parallel_size": 1,
|
|
"trust_remote_code": true,
|
|
"version": 1.0
|
|
},
|
|
"metric_list": [
|
|
{
|
|
"aggregation": "mean",
|
|
"higher_is_better": true,
|
|
"metric": "acc"
|
|
}
|
|
],
|
|
"num_fewshot": 0,
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"target_delimiter": " ",
|
|
"task": "mmlu_logical_fallacies",
|
|
"task_alias": "logical_fallacies",
|
|
"test_split": "test",
|
|
"unsafe_code": false
|
|
},
|
|
"mmlu_machine_learning": {
|
|
"dataset_name": "machine_learning",
|
|
"dataset_path": "cais/mmlu",
|
|
"description": "The following are multiple choice questions (with answers) about machine learning.\n\n",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_config": {
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_indices": null,
|
|
"gen_prefix": null,
|
|
"process_docs": null,
|
|
"sampler": "first_n",
|
|
"samples": null,
|
|
"split": "dev",
|
|
"target_delimiter": " "
|
|
},
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_split": "dev",
|
|
"metadata": {
|
|
"config_source": "[redacted]",
|
|
"dtype": "bfloat16",
|
|
"gpu_memory_utilization": 0.82,
|
|
"pretrained": "[redacted]",
|
|
"tensor_parallel_size": 1,
|
|
"trust_remote_code": true,
|
|
"version": 1.0
|
|
},
|
|
"metric_list": [
|
|
{
|
|
"aggregation": "mean",
|
|
"higher_is_better": true,
|
|
"metric": "acc"
|
|
}
|
|
],
|
|
"num_fewshot": 0,
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"target_delimiter": " ",
|
|
"task": "mmlu_machine_learning",
|
|
"task_alias": "machine_learning",
|
|
"test_split": "test",
|
|
"unsafe_code": false
|
|
},
|
|
"mmlu_management": {
|
|
"dataset_name": "management",
|
|
"dataset_path": "cais/mmlu",
|
|
"description": "The following are multiple choice questions (with answers) about management.\n\n",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_config": {
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_indices": null,
|
|
"gen_prefix": null,
|
|
"process_docs": null,
|
|
"sampler": "first_n",
|
|
"samples": null,
|
|
"split": "dev",
|
|
"target_delimiter": " "
|
|
},
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_split": "dev",
|
|
"metadata": {
|
|
"config_source": "[redacted]",
|
|
"dtype": "bfloat16",
|
|
"gpu_memory_utilization": 0.82,
|
|
"pretrained": "[redacted]",
|
|
"tensor_parallel_size": 1,
|
|
"trust_remote_code": true,
|
|
"version": 1.0
|
|
},
|
|
"metric_list": [
|
|
{
|
|
"aggregation": "mean",
|
|
"higher_is_better": true,
|
|
"metric": "acc"
|
|
}
|
|
],
|
|
"num_fewshot": 0,
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"target_delimiter": " ",
|
|
"task": "mmlu_management",
|
|
"task_alias": "management",
|
|
"test_split": "test",
|
|
"unsafe_code": false
|
|
},
|
|
"mmlu_marketing": {
|
|
"dataset_name": "marketing",
|
|
"dataset_path": "cais/mmlu",
|
|
"description": "The following are multiple choice questions (with answers) about marketing.\n\n",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_config": {
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_indices": null,
|
|
"gen_prefix": null,
|
|
"process_docs": null,
|
|
"sampler": "first_n",
|
|
"samples": null,
|
|
"split": "dev",
|
|
"target_delimiter": " "
|
|
},
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_split": "dev",
|
|
"metadata": {
|
|
"config_source": "[redacted]",
|
|
"dtype": "bfloat16",
|
|
"gpu_memory_utilization": 0.82,
|
|
"pretrained": "[redacted]",
|
|
"tensor_parallel_size": 1,
|
|
"trust_remote_code": true,
|
|
"version": 1.0
|
|
},
|
|
"metric_list": [
|
|
{
|
|
"aggregation": "mean",
|
|
"higher_is_better": true,
|
|
"metric": "acc"
|
|
}
|
|
],
|
|
"num_fewshot": 0,
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"target_delimiter": " ",
|
|
"task": "mmlu_marketing",
|
|
"task_alias": "marketing",
|
|
"test_split": "test",
|
|
"unsafe_code": false
|
|
},
|
|
"mmlu_medical_genetics": {
|
|
"dataset_name": "medical_genetics",
|
|
"dataset_path": "cais/mmlu",
|
|
"description": "The following are multiple choice questions (with answers) about medical genetics.\n\n",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_config": {
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_indices": null,
|
|
"gen_prefix": null,
|
|
"process_docs": null,
|
|
"sampler": "first_n",
|
|
"samples": null,
|
|
"split": "dev",
|
|
"target_delimiter": " "
|
|
},
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_split": "dev",
|
|
"metadata": {
|
|
"config_source": "[redacted]",
|
|
"dtype": "bfloat16",
|
|
"gpu_memory_utilization": 0.82,
|
|
"pretrained": "[redacted]",
|
|
"tensor_parallel_size": 1,
|
|
"trust_remote_code": true,
|
|
"version": 1.0
|
|
},
|
|
"metric_list": [
|
|
{
|
|
"aggregation": "mean",
|
|
"higher_is_better": true,
|
|
"metric": "acc"
|
|
}
|
|
],
|
|
"num_fewshot": 0,
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"target_delimiter": " ",
|
|
"task": "mmlu_medical_genetics",
|
|
"task_alias": "medical_genetics",
|
|
"test_split": "test",
|
|
"unsafe_code": false
|
|
},
|
|
"mmlu_miscellaneous": {
|
|
"dataset_name": "miscellaneous",
|
|
"dataset_path": "cais/mmlu",
|
|
"description": "The following are multiple choice questions (with answers) about miscellaneous.\n\n",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_config": {
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_indices": null,
|
|
"gen_prefix": null,
|
|
"process_docs": null,
|
|
"sampler": "first_n",
|
|
"samples": null,
|
|
"split": "dev",
|
|
"target_delimiter": " "
|
|
},
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_split": "dev",
|
|
"metadata": {
|
|
"config_source": "[redacted]",
|
|
"dtype": "bfloat16",
|
|
"gpu_memory_utilization": 0.82,
|
|
"pretrained": "[redacted]",
|
|
"tensor_parallel_size": 1,
|
|
"trust_remote_code": true,
|
|
"version": 1.0
|
|
},
|
|
"metric_list": [
|
|
{
|
|
"aggregation": "mean",
|
|
"higher_is_better": true,
|
|
"metric": "acc"
|
|
}
|
|
],
|
|
"num_fewshot": 0,
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"target_delimiter": " ",
|
|
"task": "mmlu_miscellaneous",
|
|
"task_alias": "miscellaneous",
|
|
"test_split": "test",
|
|
"unsafe_code": false
|
|
},
|
|
"mmlu_moral_disputes": {
|
|
"dataset_name": "moral_disputes",
|
|
"dataset_path": "cais/mmlu",
|
|
"description": "The following are multiple choice questions (with answers) about moral disputes.\n\n",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_config": {
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_indices": null,
|
|
"gen_prefix": null,
|
|
"process_docs": null,
|
|
"sampler": "first_n",
|
|
"samples": null,
|
|
"split": "dev",
|
|
"target_delimiter": " "
|
|
},
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_split": "dev",
|
|
"metadata": {
|
|
"config_source": "[redacted]",
|
|
"dtype": "bfloat16",
|
|
"gpu_memory_utilization": 0.82,
|
|
"pretrained": "[redacted]",
|
|
"tensor_parallel_size": 1,
|
|
"trust_remote_code": true,
|
|
"version": 1.0
|
|
},
|
|
"metric_list": [
|
|
{
|
|
"aggregation": "mean",
|
|
"higher_is_better": true,
|
|
"metric": "acc"
|
|
}
|
|
],
|
|
"num_fewshot": 0,
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"target_delimiter": " ",
|
|
"task": "mmlu_moral_disputes",
|
|
"task_alias": "moral_disputes",
|
|
"test_split": "test",
|
|
"unsafe_code": false
|
|
},
|
|
"mmlu_moral_scenarios": {
|
|
"dataset_name": "moral_scenarios",
|
|
"dataset_path": "cais/mmlu",
|
|
"description": "The following are multiple choice questions (with answers) about moral scenarios.\n\n",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_config": {
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_indices": null,
|
|
"gen_prefix": null,
|
|
"process_docs": null,
|
|
"sampler": "first_n",
|
|
"samples": null,
|
|
"split": "dev",
|
|
"target_delimiter": " "
|
|
},
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_split": "dev",
|
|
"metadata": {
|
|
"config_source": "[redacted]",
|
|
"dtype": "bfloat16",
|
|
"gpu_memory_utilization": 0.82,
|
|
"pretrained": "[redacted]",
|
|
"tensor_parallel_size": 1,
|
|
"trust_remote_code": true,
|
|
"version": 1.0
|
|
},
|
|
"metric_list": [
|
|
{
|
|
"aggregation": "mean",
|
|
"higher_is_better": true,
|
|
"metric": "acc"
|
|
}
|
|
],
|
|
"num_fewshot": 0,
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"target_delimiter": " ",
|
|
"task": "mmlu_moral_scenarios",
|
|
"task_alias": "moral_scenarios",
|
|
"test_split": "test",
|
|
"unsafe_code": false
|
|
},
|
|
"mmlu_nutrition": {
|
|
"dataset_name": "nutrition",
|
|
"dataset_path": "cais/mmlu",
|
|
"description": "The following are multiple choice questions (with answers) about nutrition.\n\n",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_config": {
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_indices": null,
|
|
"gen_prefix": null,
|
|
"process_docs": null,
|
|
"sampler": "first_n",
|
|
"samples": null,
|
|
"split": "dev",
|
|
"target_delimiter": " "
|
|
},
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_split": "dev",
|
|
"metadata": {
|
|
"config_source": "[redacted]",
|
|
"dtype": "bfloat16",
|
|
"gpu_memory_utilization": 0.82,
|
|
"pretrained": "[redacted]",
|
|
"tensor_parallel_size": 1,
|
|
"trust_remote_code": true,
|
|
"version": 1.0
|
|
},
|
|
"metric_list": [
|
|
{
|
|
"aggregation": "mean",
|
|
"higher_is_better": true,
|
|
"metric": "acc"
|
|
}
|
|
],
|
|
"num_fewshot": 0,
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"target_delimiter": " ",
|
|
"task": "mmlu_nutrition",
|
|
"task_alias": "nutrition",
|
|
"test_split": "test",
|
|
"unsafe_code": false
|
|
},
|
|
"mmlu_philosophy": {
|
|
"dataset_name": "philosophy",
|
|
"dataset_path": "cais/mmlu",
|
|
"description": "The following are multiple choice questions (with answers) about philosophy.\n\n",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_config": {
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_indices": null,
|
|
"gen_prefix": null,
|
|
"process_docs": null,
|
|
"sampler": "first_n",
|
|
"samples": null,
|
|
"split": "dev",
|
|
"target_delimiter": " "
|
|
},
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_split": "dev",
|
|
"metadata": {
|
|
"config_source": "[redacted]",
|
|
"dtype": "bfloat16",
|
|
"gpu_memory_utilization": 0.82,
|
|
"pretrained": "[redacted]",
|
|
"tensor_parallel_size": 1,
|
|
"trust_remote_code": true,
|
|
"version": 1.0
|
|
},
|
|
"metric_list": [
|
|
{
|
|
"aggregation": "mean",
|
|
"higher_is_better": true,
|
|
"metric": "acc"
|
|
}
|
|
],
|
|
"num_fewshot": 0,
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"target_delimiter": " ",
|
|
"task": "mmlu_philosophy",
|
|
"task_alias": "philosophy",
|
|
"test_split": "test",
|
|
"unsafe_code": false
|
|
},
|
|
"mmlu_prehistory": {
|
|
"dataset_name": "prehistory",
|
|
"dataset_path": "cais/mmlu",
|
|
"description": "The following are multiple choice questions (with answers) about prehistory.\n\n",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_config": {
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_indices": null,
|
|
"gen_prefix": null,
|
|
"process_docs": null,
|
|
"sampler": "first_n",
|
|
"samples": null,
|
|
"split": "dev",
|
|
"target_delimiter": " "
|
|
},
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_split": "dev",
|
|
"metadata": {
|
|
"config_source": "[redacted]",
|
|
"dtype": "bfloat16",
|
|
"gpu_memory_utilization": 0.82,
|
|
"pretrained": "[redacted]",
|
|
"tensor_parallel_size": 1,
|
|
"trust_remote_code": true,
|
|
"version": 1.0
|
|
},
|
|
"metric_list": [
|
|
{
|
|
"aggregation": "mean",
|
|
"higher_is_better": true,
|
|
"metric": "acc"
|
|
}
|
|
],
|
|
"num_fewshot": 0,
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"target_delimiter": " ",
|
|
"task": "mmlu_prehistory",
|
|
"task_alias": "prehistory",
|
|
"test_split": "test",
|
|
"unsafe_code": false
|
|
},
|
|
"mmlu_professional_accounting": {
|
|
"dataset_name": "professional_accounting",
|
|
"dataset_path": "cais/mmlu",
|
|
"description": "The following are multiple choice questions (with answers) about professional accounting.\n\n",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_config": {
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_indices": null,
|
|
"gen_prefix": null,
|
|
"process_docs": null,
|
|
"sampler": "first_n",
|
|
"samples": null,
|
|
"split": "dev",
|
|
"target_delimiter": " "
|
|
},
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_split": "dev",
|
|
"metadata": {
|
|
"config_source": "[redacted]",
|
|
"dtype": "bfloat16",
|
|
"gpu_memory_utilization": 0.82,
|
|
"pretrained": "[redacted]",
|
|
"tensor_parallel_size": 1,
|
|
"trust_remote_code": true,
|
|
"version": 1.0
|
|
},
|
|
"metric_list": [
|
|
{
|
|
"aggregation": "mean",
|
|
"higher_is_better": true,
|
|
"metric": "acc"
|
|
}
|
|
],
|
|
"num_fewshot": 0,
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"target_delimiter": " ",
|
|
"task": "mmlu_professional_accounting",
|
|
"task_alias": "professional_accounting",
|
|
"test_split": "test",
|
|
"unsafe_code": false
|
|
},
|
|
"mmlu_professional_law": {
|
|
"dataset_name": "professional_law",
|
|
"dataset_path": "cais/mmlu",
|
|
"description": "The following are multiple choice questions (with answers) about professional law.\n\n",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_config": {
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_indices": null,
|
|
"gen_prefix": null,
|
|
"process_docs": null,
|
|
"sampler": "first_n",
|
|
"samples": null,
|
|
"split": "dev",
|
|
"target_delimiter": " "
|
|
},
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_split": "dev",
|
|
"metadata": {
|
|
"config_source": "[redacted]",
|
|
"dtype": "bfloat16",
|
|
"gpu_memory_utilization": 0.82,
|
|
"pretrained": "[redacted]",
|
|
"tensor_parallel_size": 1,
|
|
"trust_remote_code": true,
|
|
"version": 1.0
|
|
},
|
|
"metric_list": [
|
|
{
|
|
"aggregation": "mean",
|
|
"higher_is_better": true,
|
|
"metric": "acc"
|
|
}
|
|
],
|
|
"num_fewshot": 0,
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"target_delimiter": " ",
|
|
"task": "mmlu_professional_law",
|
|
"task_alias": "professional_law",
|
|
"test_split": "test",
|
|
"unsafe_code": false
|
|
},
|
|
"mmlu_professional_medicine": {
|
|
"dataset_name": "professional_medicine",
|
|
"dataset_path": "cais/mmlu",
|
|
"description": "The following are multiple choice questions (with answers) about professional medicine.\n\n",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_config": {
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_indices": null,
|
|
"gen_prefix": null,
|
|
"process_docs": null,
|
|
"sampler": "first_n",
|
|
"samples": null,
|
|
"split": "dev",
|
|
"target_delimiter": " "
|
|
},
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_split": "dev",
|
|
"metadata": {
|
|
"config_source": "[redacted]",
|
|
"dtype": "bfloat16",
|
|
"gpu_memory_utilization": 0.82,
|
|
"pretrained": "[redacted]",
|
|
"tensor_parallel_size": 1,
|
|
"trust_remote_code": true,
|
|
"version": 1.0
|
|
},
|
|
"metric_list": [
|
|
{
|
|
"aggregation": "mean",
|
|
"higher_is_better": true,
|
|
"metric": "acc"
|
|
}
|
|
],
|
|
"num_fewshot": 0,
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"target_delimiter": " ",
|
|
"task": "mmlu_professional_medicine",
|
|
"task_alias": "professional_medicine",
|
|
"test_split": "test",
|
|
"unsafe_code": false
|
|
},
|
|
"mmlu_professional_psychology": {
|
|
"dataset_name": "professional_psychology",
|
|
"dataset_path": "cais/mmlu",
|
|
"description": "The following are multiple choice questions (with answers) about professional psychology.\n\n",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_config": {
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_indices": null,
|
|
"gen_prefix": null,
|
|
"process_docs": null,
|
|
"sampler": "first_n",
|
|
"samples": null,
|
|
"split": "dev",
|
|
"target_delimiter": " "
|
|
},
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_split": "dev",
|
|
"metadata": {
|
|
"config_source": "[redacted]",
|
|
"dtype": "bfloat16",
|
|
"gpu_memory_utilization": 0.82,
|
|
"pretrained": "[redacted]",
|
|
"tensor_parallel_size": 1,
|
|
"trust_remote_code": true,
|
|
"version": 1.0
|
|
},
|
|
"metric_list": [
|
|
{
|
|
"aggregation": "mean",
|
|
"higher_is_better": true,
|
|
"metric": "acc"
|
|
}
|
|
],
|
|
"num_fewshot": 0,
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"target_delimiter": " ",
|
|
"task": "mmlu_professional_psychology",
|
|
"task_alias": "professional_psychology",
|
|
"test_split": "test",
|
|
"unsafe_code": false
|
|
},
|
|
"mmlu_public_relations": {
|
|
"dataset_name": "public_relations",
|
|
"dataset_path": "cais/mmlu",
|
|
"description": "The following are multiple choice questions (with answers) about public relations.\n\n",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_config": {
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_indices": null,
|
|
"gen_prefix": null,
|
|
"process_docs": null,
|
|
"sampler": "first_n",
|
|
"samples": null,
|
|
"split": "dev",
|
|
"target_delimiter": " "
|
|
},
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_split": "dev",
|
|
"metadata": {
|
|
"config_source": "[redacted]",
|
|
"dtype": "bfloat16",
|
|
"gpu_memory_utilization": 0.82,
|
|
"pretrained": "[redacted]",
|
|
"tensor_parallel_size": 1,
|
|
"trust_remote_code": true,
|
|
"version": 1.0
|
|
},
|
|
"metric_list": [
|
|
{
|
|
"aggregation": "mean",
|
|
"higher_is_better": true,
|
|
"metric": "acc"
|
|
}
|
|
],
|
|
"num_fewshot": 0,
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"target_delimiter": " ",
|
|
"task": "mmlu_public_relations",
|
|
"task_alias": "public_relations",
|
|
"test_split": "test",
|
|
"unsafe_code": false
|
|
},
|
|
"mmlu_security_studies": {
|
|
"dataset_name": "security_studies",
|
|
"dataset_path": "cais/mmlu",
|
|
"description": "The following are multiple choice questions (with answers) about security studies.\n\n",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_config": {
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_indices": null,
|
|
"gen_prefix": null,
|
|
"process_docs": null,
|
|
"sampler": "first_n",
|
|
"samples": null,
|
|
"split": "dev",
|
|
"target_delimiter": " "
|
|
},
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_split": "dev",
|
|
"metadata": {
|
|
"config_source": "[redacted]",
|
|
"dtype": "bfloat16",
|
|
"gpu_memory_utilization": 0.82,
|
|
"pretrained": "[redacted]",
|
|
"tensor_parallel_size": 1,
|
|
"trust_remote_code": true,
|
|
"version": 1.0
|
|
},
|
|
"metric_list": [
|
|
{
|
|
"aggregation": "mean",
|
|
"higher_is_better": true,
|
|
"metric": "acc"
|
|
}
|
|
],
|
|
"num_fewshot": 0,
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"target_delimiter": " ",
|
|
"task": "mmlu_security_studies",
|
|
"task_alias": "security_studies",
|
|
"test_split": "test",
|
|
"unsafe_code": false
|
|
},
|
|
"mmlu_sociology": {
|
|
"dataset_name": "sociology",
|
|
"dataset_path": "cais/mmlu",
|
|
"description": "The following are multiple choice questions (with answers) about sociology.\n\n",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_config": {
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_indices": null,
|
|
"gen_prefix": null,
|
|
"process_docs": null,
|
|
"sampler": "first_n",
|
|
"samples": null,
|
|
"split": "dev",
|
|
"target_delimiter": " "
|
|
},
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_split": "dev",
|
|
"metadata": {
|
|
"config_source": "[redacted]",
|
|
"dtype": "bfloat16",
|
|
"gpu_memory_utilization": 0.82,
|
|
"pretrained": "[redacted]",
|
|
"tensor_parallel_size": 1,
|
|
"trust_remote_code": true,
|
|
"version": 1.0
|
|
},
|
|
"metric_list": [
|
|
{
|
|
"aggregation": "mean",
|
|
"higher_is_better": true,
|
|
"metric": "acc"
|
|
}
|
|
],
|
|
"num_fewshot": 0,
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"target_delimiter": " ",
|
|
"task": "mmlu_sociology",
|
|
"task_alias": "sociology",
|
|
"test_split": "test",
|
|
"unsafe_code": false
|
|
},
|
|
"mmlu_us_foreign_policy": {
|
|
"dataset_name": "us_foreign_policy",
|
|
"dataset_path": "cais/mmlu",
|
|
"description": "The following are multiple choice questions (with answers) about us foreign policy.\n\n",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_config": {
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_indices": null,
|
|
"gen_prefix": null,
|
|
"process_docs": null,
|
|
"sampler": "first_n",
|
|
"samples": null,
|
|
"split": "dev",
|
|
"target_delimiter": " "
|
|
},
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_split": "dev",
|
|
"metadata": {
|
|
"config_source": "[redacted]",
|
|
"dtype": "bfloat16",
|
|
"gpu_memory_utilization": 0.82,
|
|
"pretrained": "[redacted]",
|
|
"tensor_parallel_size": 1,
|
|
"trust_remote_code": true,
|
|
"version": 1.0
|
|
},
|
|
"metric_list": [
|
|
{
|
|
"aggregation": "mean",
|
|
"higher_is_better": true,
|
|
"metric": "acc"
|
|
}
|
|
],
|
|
"num_fewshot": 0,
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"target_delimiter": " ",
|
|
"task": "mmlu_us_foreign_policy",
|
|
"task_alias": "us_foreign_policy",
|
|
"test_split": "test",
|
|
"unsafe_code": false
|
|
},
|
|
"mmlu_virology": {
|
|
"dataset_name": "virology",
|
|
"dataset_path": "cais/mmlu",
|
|
"description": "The following are multiple choice questions (with answers) about virology.\n\n",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_config": {
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_indices": null,
|
|
"gen_prefix": null,
|
|
"process_docs": null,
|
|
"sampler": "first_n",
|
|
"samples": null,
|
|
"split": "dev",
|
|
"target_delimiter": " "
|
|
},
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_split": "dev",
|
|
"metadata": {
|
|
"config_source": "[redacted]",
|
|
"dtype": "bfloat16",
|
|
"gpu_memory_utilization": 0.82,
|
|
"pretrained": "[redacted]",
|
|
"tensor_parallel_size": 1,
|
|
"trust_remote_code": true,
|
|
"version": 1.0
|
|
},
|
|
"metric_list": [
|
|
{
|
|
"aggregation": "mean",
|
|
"higher_is_better": true,
|
|
"metric": "acc"
|
|
}
|
|
],
|
|
"num_fewshot": 0,
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"target_delimiter": " ",
|
|
"task": "mmlu_virology",
|
|
"task_alias": "virology",
|
|
"test_split": "test",
|
|
"unsafe_code": false
|
|
},
|
|
"mmlu_world_religions": {
|
|
"dataset_name": "world_religions",
|
|
"dataset_path": "cais/mmlu",
|
|
"description": "The following are multiple choice questions (with answers) about world religions.\n\n",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_config": {
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"doc_to_target": "answer",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_indices": null,
|
|
"gen_prefix": null,
|
|
"process_docs": null,
|
|
"sampler": "first_n",
|
|
"samples": null,
|
|
"split": "dev",
|
|
"target_delimiter": " "
|
|
},
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_split": "dev",
|
|
"metadata": {
|
|
"config_source": "[redacted]",
|
|
"dtype": "bfloat16",
|
|
"gpu_memory_utilization": 0.82,
|
|
"pretrained": "[redacted]",
|
|
"tensor_parallel_size": 1,
|
|
"trust_remote_code": true,
|
|
"version": 1.0
|
|
},
|
|
"metric_list": [
|
|
{
|
|
"aggregation": "mean",
|
|
"higher_is_better": true,
|
|
"metric": "acc"
|
|
}
|
|
],
|
|
"num_fewshot": 0,
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"target_delimiter": " ",
|
|
"task": "mmlu_world_religions",
|
|
"task_alias": "world_religions",
|
|
"test_split": "test",
|
|
"unsafe_code": false
|
|
}
|
|
},
|
|
"date": 1781174539.5106196,
|
|
"eot_token_id": 151645,
|
|
"fewshot_as_multiturn": true,
|
|
"git_hash": null,
|
|
"group_subtasks": {
|
|
"mmlu": [
|
|
"mmlu_stem",
|
|
"mmlu_other",
|
|
"mmlu_social_sciences",
|
|
"mmlu_humanities"
|
|
],
|
|
"mmlu_humanities": [
|
|
"mmlu_formal_logic",
|
|
"mmlu_high_school_european_history",
|
|
"mmlu_high_school_us_history",
|
|
"mmlu_high_school_world_history",
|
|
"mmlu_international_law",
|
|
"mmlu_jurisprudence",
|
|
"mmlu_logical_fallacies",
|
|
"mmlu_moral_disputes",
|
|
"mmlu_moral_scenarios",
|
|
"mmlu_philosophy",
|
|
"mmlu_prehistory",
|
|
"mmlu_professional_law",
|
|
"mmlu_world_religions"
|
|
],
|
|
"mmlu_other": [
|
|
"mmlu_business_ethics",
|
|
"mmlu_clinical_knowledge",
|
|
"mmlu_college_medicine",
|
|
"mmlu_global_facts",
|
|
"mmlu_human_aging",
|
|
"mmlu_management",
|
|
"mmlu_marketing",
|
|
"mmlu_medical_genetics",
|
|
"mmlu_miscellaneous",
|
|
"mmlu_nutrition",
|
|
"mmlu_professional_accounting",
|
|
"mmlu_professional_medicine",
|
|
"mmlu_virology"
|
|
],
|
|
"mmlu_social_sciences": [
|
|
"mmlu_econometrics",
|
|
"mmlu_high_school_geography",
|
|
"mmlu_high_school_government_and_politics",
|
|
"mmlu_high_school_macroeconomics",
|
|
"mmlu_high_school_microeconomics",
|
|
"mmlu_high_school_psychology",
|
|
"mmlu_human_sexuality",
|
|
"mmlu_professional_psychology",
|
|
"mmlu_public_relations",
|
|
"mmlu_security_studies",
|
|
"mmlu_sociology",
|
|
"mmlu_us_foreign_policy"
|
|
],
|
|
"mmlu_stem": [
|
|
"mmlu_abstract_algebra",
|
|
"mmlu_anatomy",
|
|
"mmlu_astronomy",
|
|
"mmlu_college_biology",
|
|
"mmlu_college_chemistry",
|
|
"mmlu_college_computer_science",
|
|
"mmlu_college_mathematics",
|
|
"mmlu_college_physics",
|
|
"mmlu_computer_security",
|
|
"mmlu_conceptual_physics",
|
|
"mmlu_electrical_engineering",
|
|
"mmlu_elementary_mathematics",
|
|
"mmlu_high_school_biology",
|
|
"mmlu_high_school_chemistry",
|
|
"mmlu_high_school_computer_science",
|
|
"mmlu_high_school_mathematics",
|
|
"mmlu_high_school_physics",
|
|
"mmlu_high_school_statistics",
|
|
"mmlu_machine_learning"
|
|
]
|
|
},
|
|
"groups": {
|
|
"mmlu": {
|
|
"acc,none": 0.6575986326734083,
|
|
"acc_stderr,none": 0.0036785245967130517,
|
|
"alias": "mmlu",
|
|
"name": "mmlu",
|
|
"sample_count": {
|
|
"acc,none": 14042
|
|
},
|
|
"sample_len": 14042
|
|
},
|
|
"mmlu_humanities": {
|
|
"acc,none": 0.5883103081827843,
|
|
"acc_stderr,none": 0.006616678303159137,
|
|
"alias": "humanities",
|
|
"name": "mmlu_humanities",
|
|
"sample_count": {
|
|
"acc,none": 4705
|
|
},
|
|
"sample_len": 4705
|
|
},
|
|
"mmlu_other": {
|
|
"acc,none": 0.7219182491149019,
|
|
"acc_stderr,none": 0.007740746200087848,
|
|
"alias": "other",
|
|
"name": "mmlu_other",
|
|
"sample_count": {
|
|
"acc,none": 3107
|
|
},
|
|
"sample_len": 3107
|
|
},
|
|
"mmlu_social_sciences": {
|
|
"acc,none": 0.7897302567435814,
|
|
"acc_stderr,none": 0.007206549343813523,
|
|
"alias": "social sciences",
|
|
"name": "mmlu_social_sciences",
|
|
"sample_count": {
|
|
"acc,none": 3077
|
|
},
|
|
"sample_len": 3077
|
|
},
|
|
"mmlu_stem": {
|
|
"acc,none": 0.5686647637170948,
|
|
"acc_stderr,none": 0.007952947130384178,
|
|
"alias": "stem",
|
|
"name": "mmlu_stem",
|
|
"sample_count": {
|
|
"acc,none": 3153
|
|
},
|
|
"sample_len": 3153
|
|
}
|
|
},
|
|
"higher_is_better": {
|
|
"mmlu_abstract_algebra": {
|
|
"acc": true
|
|
},
|
|
"mmlu_anatomy": {
|
|
"acc": true
|
|
},
|
|
"mmlu_astronomy": {
|
|
"acc": true
|
|
},
|
|
"mmlu_business_ethics": {
|
|
"acc": true
|
|
},
|
|
"mmlu_clinical_knowledge": {
|
|
"acc": true
|
|
},
|
|
"mmlu_college_biology": {
|
|
"acc": true
|
|
},
|
|
"mmlu_college_chemistry": {
|
|
"acc": true
|
|
},
|
|
"mmlu_college_computer_science": {
|
|
"acc": true
|
|
},
|
|
"mmlu_college_mathematics": {
|
|
"acc": true
|
|
},
|
|
"mmlu_college_medicine": {
|
|
"acc": true
|
|
},
|
|
"mmlu_college_physics": {
|
|
"acc": true
|
|
},
|
|
"mmlu_computer_security": {
|
|
"acc": true
|
|
},
|
|
"mmlu_conceptual_physics": {
|
|
"acc": true
|
|
},
|
|
"mmlu_econometrics": {
|
|
"acc": true
|
|
},
|
|
"mmlu_electrical_engineering": {
|
|
"acc": true
|
|
},
|
|
"mmlu_elementary_mathematics": {
|
|
"acc": true
|
|
},
|
|
"mmlu_formal_logic": {
|
|
"acc": true
|
|
},
|
|
"mmlu_global_facts": {
|
|
"acc": true
|
|
},
|
|
"mmlu_high_school_biology": {
|
|
"acc": true
|
|
},
|
|
"mmlu_high_school_chemistry": {
|
|
"acc": true
|
|
},
|
|
"mmlu_high_school_computer_science": {
|
|
"acc": true
|
|
},
|
|
"mmlu_high_school_european_history": {
|
|
"acc": true
|
|
},
|
|
"mmlu_high_school_geography": {
|
|
"acc": true
|
|
},
|
|
"mmlu_high_school_government_and_politics": {
|
|
"acc": true
|
|
},
|
|
"mmlu_high_school_macroeconomics": {
|
|
"acc": true
|
|
},
|
|
"mmlu_high_school_mathematics": {
|
|
"acc": true
|
|
},
|
|
"mmlu_high_school_microeconomics": {
|
|
"acc": true
|
|
},
|
|
"mmlu_high_school_physics": {
|
|
"acc": true
|
|
},
|
|
"mmlu_high_school_psychology": {
|
|
"acc": true
|
|
},
|
|
"mmlu_high_school_statistics": {
|
|
"acc": true
|
|
},
|
|
"mmlu_high_school_us_history": {
|
|
"acc": true
|
|
},
|
|
"mmlu_high_school_world_history": {
|
|
"acc": true
|
|
},
|
|
"mmlu_human_aging": {
|
|
"acc": true
|
|
},
|
|
"mmlu_human_sexuality": {
|
|
"acc": true
|
|
},
|
|
"mmlu_humanities": {
|
|
"acc": true
|
|
},
|
|
"mmlu_international_law": {
|
|
"acc": true
|
|
},
|
|
"mmlu_jurisprudence": {
|
|
"acc": true
|
|
},
|
|
"mmlu_logical_fallacies": {
|
|
"acc": true
|
|
},
|
|
"mmlu_machine_learning": {
|
|
"acc": true
|
|
},
|
|
"mmlu_management": {
|
|
"acc": true
|
|
},
|
|
"mmlu_marketing": {
|
|
"acc": true
|
|
},
|
|
"mmlu_medical_genetics": {
|
|
"acc": true
|
|
},
|
|
"mmlu_miscellaneous": {
|
|
"acc": true
|
|
},
|
|
"mmlu_moral_disputes": {
|
|
"acc": true
|
|
},
|
|
"mmlu_moral_scenarios": {
|
|
"acc": true
|
|
},
|
|
"mmlu_nutrition": {
|
|
"acc": true
|
|
},
|
|
"mmlu_other": {
|
|
"acc": true
|
|
},
|
|
"mmlu_philosophy": {
|
|
"acc": true
|
|
},
|
|
"mmlu_prehistory": {
|
|
"acc": true
|
|
},
|
|
"mmlu_professional_accounting": {
|
|
"acc": true
|
|
},
|
|
"mmlu_professional_law": {
|
|
"acc": true
|
|
},
|
|
"mmlu_professional_medicine": {
|
|
"acc": true
|
|
},
|
|
"mmlu_professional_psychology": {
|
|
"acc": true
|
|
},
|
|
"mmlu_public_relations": {
|
|
"acc": true
|
|
},
|
|
"mmlu_security_studies": {
|
|
"acc": true
|
|
},
|
|
"mmlu_social_sciences": {
|
|
"acc": true
|
|
},
|
|
"mmlu_sociology": {
|
|
"acc": true
|
|
},
|
|
"mmlu_stem": {
|
|
"acc": true
|
|
},
|
|
"mmlu_us_foreign_policy": {
|
|
"acc": true
|
|
},
|
|
"mmlu_virology": {
|
|
"acc": true
|
|
},
|
|
"mmlu_world_religions": {
|
|
"acc": true
|
|
}
|
|
},
|
|
"lm_eval_version": "0.4.12",
|
|
"max_length": 262144,
|
|
"model_name": "kosa-labs/kosa-4B-it-v1",
|
|
"model_name_sanitized": "kosa-labs/kosa-4B-it-v1",
|
|
"model_source": "kosa-labs/kosa-4B-it-v1",
|
|
"n-samples": {
|
|
"mmlu_abstract_algebra": {
|
|
"effective": 100,
|
|
"original": 100
|
|
},
|
|
"mmlu_anatomy": {
|
|
"effective": 135,
|
|
"original": 135
|
|
},
|
|
"mmlu_astronomy": {
|
|
"effective": 152,
|
|
"original": 152
|
|
},
|
|
"mmlu_business_ethics": {
|
|
"effective": 100,
|
|
"original": 100
|
|
},
|
|
"mmlu_clinical_knowledge": {
|
|
"effective": 265,
|
|
"original": 265
|
|
},
|
|
"mmlu_college_biology": {
|
|
"effective": 144,
|
|
"original": 144
|
|
},
|
|
"mmlu_college_chemistry": {
|
|
"effective": 100,
|
|
"original": 100
|
|
},
|
|
"mmlu_college_computer_science": {
|
|
"effective": 100,
|
|
"original": 100
|
|
},
|
|
"mmlu_college_mathematics": {
|
|
"effective": 100,
|
|
"original": 100
|
|
},
|
|
"mmlu_college_medicine": {
|
|
"effective": 173,
|
|
"original": 173
|
|
},
|
|
"mmlu_college_physics": {
|
|
"effective": 102,
|
|
"original": 102
|
|
},
|
|
"mmlu_computer_security": {
|
|
"effective": 100,
|
|
"original": 100
|
|
},
|
|
"mmlu_conceptual_physics": {
|
|
"effective": 235,
|
|
"original": 235
|
|
},
|
|
"mmlu_econometrics": {
|
|
"effective": 114,
|
|
"original": 114
|
|
},
|
|
"mmlu_electrical_engineering": {
|
|
"effective": 145,
|
|
"original": 145
|
|
},
|
|
"mmlu_elementary_mathematics": {
|
|
"effective": 378,
|
|
"original": 378
|
|
},
|
|
"mmlu_formal_logic": {
|
|
"effective": 126,
|
|
"original": 126
|
|
},
|
|
"mmlu_global_facts": {
|
|
"effective": 100,
|
|
"original": 100
|
|
},
|
|
"mmlu_high_school_biology": {
|
|
"effective": 310,
|
|
"original": 310
|
|
},
|
|
"mmlu_high_school_chemistry": {
|
|
"effective": 203,
|
|
"original": 203
|
|
},
|
|
"mmlu_high_school_computer_science": {
|
|
"effective": 100,
|
|
"original": 100
|
|
},
|
|
"mmlu_high_school_european_history": {
|
|
"effective": 165,
|
|
"original": 165
|
|
},
|
|
"mmlu_high_school_geography": {
|
|
"effective": 198,
|
|
"original": 198
|
|
},
|
|
"mmlu_high_school_government_and_politics": {
|
|
"effective": 193,
|
|
"original": 193
|
|
},
|
|
"mmlu_high_school_macroeconomics": {
|
|
"effective": 390,
|
|
"original": 390
|
|
},
|
|
"mmlu_high_school_mathematics": {
|
|
"effective": 270,
|
|
"original": 270
|
|
},
|
|
"mmlu_high_school_microeconomics": {
|
|
"effective": 238,
|
|
"original": 238
|
|
},
|
|
"mmlu_high_school_physics": {
|
|
"effective": 151,
|
|
"original": 151
|
|
},
|
|
"mmlu_high_school_psychology": {
|
|
"effective": 545,
|
|
"original": 545
|
|
},
|
|
"mmlu_high_school_statistics": {
|
|
"effective": 216,
|
|
"original": 216
|
|
},
|
|
"mmlu_high_school_us_history": {
|
|
"effective": 204,
|
|
"original": 204
|
|
},
|
|
"mmlu_high_school_world_history": {
|
|
"effective": 237,
|
|
"original": 237
|
|
},
|
|
"mmlu_human_aging": {
|
|
"effective": 223,
|
|
"original": 223
|
|
},
|
|
"mmlu_human_sexuality": {
|
|
"effective": 131,
|
|
"original": 131
|
|
},
|
|
"mmlu_international_law": {
|
|
"effective": 121,
|
|
"original": 121
|
|
},
|
|
"mmlu_jurisprudence": {
|
|
"effective": 108,
|
|
"original": 108
|
|
},
|
|
"mmlu_logical_fallacies": {
|
|
"effective": 163,
|
|
"original": 163
|
|
},
|
|
"mmlu_machine_learning": {
|
|
"effective": 112,
|
|
"original": 112
|
|
},
|
|
"mmlu_management": {
|
|
"effective": 103,
|
|
"original": 103
|
|
},
|
|
"mmlu_marketing": {
|
|
"effective": 234,
|
|
"original": 234
|
|
},
|
|
"mmlu_medical_genetics": {
|
|
"effective": 100,
|
|
"original": 100
|
|
},
|
|
"mmlu_miscellaneous": {
|
|
"effective": 783,
|
|
"original": 783
|
|
},
|
|
"mmlu_moral_disputes": {
|
|
"effective": 346,
|
|
"original": 346
|
|
},
|
|
"mmlu_moral_scenarios": {
|
|
"effective": 895,
|
|
"original": 895
|
|
},
|
|
"mmlu_nutrition": {
|
|
"effective": 306,
|
|
"original": 306
|
|
},
|
|
"mmlu_philosophy": {
|
|
"effective": 311,
|
|
"original": 311
|
|
},
|
|
"mmlu_prehistory": {
|
|
"effective": 324,
|
|
"original": 324
|
|
},
|
|
"mmlu_professional_accounting": {
|
|
"effective": 282,
|
|
"original": 282
|
|
},
|
|
"mmlu_professional_law": {
|
|
"effective": 1534,
|
|
"original": 1534
|
|
},
|
|
"mmlu_professional_medicine": {
|
|
"effective": 272,
|
|
"original": 272
|
|
},
|
|
"mmlu_professional_psychology": {
|
|
"effective": 612,
|
|
"original": 612
|
|
},
|
|
"mmlu_public_relations": {
|
|
"effective": 110,
|
|
"original": 110
|
|
},
|
|
"mmlu_security_studies": {
|
|
"effective": 245,
|
|
"original": 245
|
|
},
|
|
"mmlu_sociology": {
|
|
"effective": 201,
|
|
"original": 201
|
|
},
|
|
"mmlu_us_foreign_policy": {
|
|
"effective": 100,
|
|
"original": 100
|
|
},
|
|
"mmlu_virology": {
|
|
"effective": 166,
|
|
"original": 166
|
|
},
|
|
"mmlu_world_religions": {
|
|
"effective": 171,
|
|
"original": 171
|
|
}
|
|
},
|
|
"n-shot": {
|
|
"mmlu_abstract_algebra": 0,
|
|
"mmlu_anatomy": 0,
|
|
"mmlu_astronomy": 0,
|
|
"mmlu_business_ethics": 0,
|
|
"mmlu_clinical_knowledge": 0,
|
|
"mmlu_college_biology": 0,
|
|
"mmlu_college_chemistry": 0,
|
|
"mmlu_college_computer_science": 0,
|
|
"mmlu_college_mathematics": 0,
|
|
"mmlu_college_medicine": 0,
|
|
"mmlu_college_physics": 0,
|
|
"mmlu_computer_security": 0,
|
|
"mmlu_conceptual_physics": 0,
|
|
"mmlu_econometrics": 0,
|
|
"mmlu_electrical_engineering": 0,
|
|
"mmlu_elementary_mathematics": 0,
|
|
"mmlu_formal_logic": 0,
|
|
"mmlu_global_facts": 0,
|
|
"mmlu_high_school_biology": 0,
|
|
"mmlu_high_school_chemistry": 0,
|
|
"mmlu_high_school_computer_science": 0,
|
|
"mmlu_high_school_european_history": 0,
|
|
"mmlu_high_school_geography": 0,
|
|
"mmlu_high_school_government_and_politics": 0,
|
|
"mmlu_high_school_macroeconomics": 0,
|
|
"mmlu_high_school_mathematics": 0,
|
|
"mmlu_high_school_microeconomics": 0,
|
|
"mmlu_high_school_physics": 0,
|
|
"mmlu_high_school_psychology": 0,
|
|
"mmlu_high_school_statistics": 0,
|
|
"mmlu_high_school_us_history": 0,
|
|
"mmlu_high_school_world_history": 0,
|
|
"mmlu_human_aging": 0,
|
|
"mmlu_human_sexuality": 0,
|
|
"mmlu_humanities": 0,
|
|
"mmlu_international_law": 0,
|
|
"mmlu_jurisprudence": 0,
|
|
"mmlu_logical_fallacies": 0,
|
|
"mmlu_machine_learning": 0,
|
|
"mmlu_management": 0,
|
|
"mmlu_marketing": 0,
|
|
"mmlu_medical_genetics": 0,
|
|
"mmlu_miscellaneous": 0,
|
|
"mmlu_moral_disputes": 0,
|
|
"mmlu_moral_scenarios": 0,
|
|
"mmlu_nutrition": 0,
|
|
"mmlu_other": 0,
|
|
"mmlu_philosophy": 0,
|
|
"mmlu_prehistory": 0,
|
|
"mmlu_professional_accounting": 0,
|
|
"mmlu_professional_law": 0,
|
|
"mmlu_professional_medicine": 0,
|
|
"mmlu_professional_psychology": 0,
|
|
"mmlu_public_relations": 0,
|
|
"mmlu_security_studies": 0,
|
|
"mmlu_social_sciences": 0,
|
|
"mmlu_sociology": 0,
|
|
"mmlu_stem": 0,
|
|
"mmlu_us_foreign_policy": 0,
|
|
"mmlu_virology": 0,
|
|
"mmlu_world_religions": 0
|
|
},
|
|
"results": {
|
|
"mmlu": {
|
|
"acc,none": 0.6575986326734083,
|
|
"acc_stderr,none": 0.0036785245967130517,
|
|
"alias": "mmlu",
|
|
"name": "mmlu",
|
|
"sample_count": {
|
|
"acc,none": 14042
|
|
},
|
|
"sample_len": 14042
|
|
},
|
|
"mmlu_abstract_algebra": {
|
|
"acc,none": 0.25,
|
|
"acc_stderr,none": 0.04351941398892446,
|
|
"alias": "abstract_algebra",
|
|
"name": "mmlu_abstract_algebra",
|
|
"sample_len": 100
|
|
},
|
|
"mmlu_anatomy": {
|
|
"acc,none": 0.674074074074074,
|
|
"acc_stderr,none": 0.040491220417025006,
|
|
"alias": "anatomy",
|
|
"name": "mmlu_anatomy",
|
|
"sample_len": 135
|
|
},
|
|
"mmlu_astronomy": {
|
|
"acc,none": 0.8355263157894737,
|
|
"acc_stderr,none": 0.030167533468632726,
|
|
"alias": "astronomy",
|
|
"name": "mmlu_astronomy",
|
|
"sample_len": 152
|
|
},
|
|
"mmlu_business_ethics": {
|
|
"acc,none": 0.74,
|
|
"acc_stderr,none": 0.0440844002276808,
|
|
"alias": "business_ethics",
|
|
"name": "mmlu_business_ethics",
|
|
"sample_len": 100
|
|
},
|
|
"mmlu_clinical_knowledge": {
|
|
"acc,none": 0.7509433962264151,
|
|
"acc_stderr,none": 0.02661648298050167,
|
|
"alias": "clinical_knowledge",
|
|
"name": "mmlu_clinical_knowledge",
|
|
"sample_len": 265
|
|
},
|
|
"mmlu_college_biology": {
|
|
"acc,none": 0.8680555555555556,
|
|
"acc_stderr,none": 0.02830096838204442,
|
|
"alias": "college_biology",
|
|
"name": "mmlu_college_biology",
|
|
"sample_len": 144
|
|
},
|
|
"mmlu_college_chemistry": {
|
|
"acc,none": 0.47,
|
|
"acc_stderr,none": 0.05016135580465919,
|
|
"alias": "college_chemistry",
|
|
"name": "mmlu_college_chemistry",
|
|
"sample_len": 100
|
|
},
|
|
"mmlu_college_computer_science": {
|
|
"acc,none": 0.51,
|
|
"acc_stderr,none": 0.05024183937956913,
|
|
"alias": "college_computer_science",
|
|
"name": "mmlu_college_computer_science",
|
|
"sample_len": 100
|
|
},
|
|
"mmlu_college_mathematics": {
|
|
"acc,none": 0.24,
|
|
"acc_stderr,none": 0.04292346959909278,
|
|
"alias": "college_mathematics",
|
|
"name": "mmlu_college_mathematics",
|
|
"sample_len": 100
|
|
},
|
|
"mmlu_college_medicine": {
|
|
"acc,none": 0.6705202312138728,
|
|
"acc_stderr,none": 0.03583901754736415,
|
|
"alias": "college_medicine",
|
|
"name": "mmlu_college_medicine",
|
|
"sample_len": 173
|
|
},
|
|
"mmlu_college_physics": {
|
|
"acc,none": 0.39215686274509803,
|
|
"acc_stderr,none": 0.04858083574266346,
|
|
"alias": "college_physics",
|
|
"name": "mmlu_college_physics",
|
|
"sample_len": 102
|
|
},
|
|
"mmlu_computer_security": {
|
|
"acc,none": 0.79,
|
|
"acc_stderr,none": 0.040936018074033236,
|
|
"alias": "computer_security",
|
|
"name": "mmlu_computer_security",
|
|
"sample_len": 100
|
|
},
|
|
"mmlu_conceptual_physics": {
|
|
"acc,none": 0.774468085106383,
|
|
"acc_stderr,none": 0.02732107841738755,
|
|
"alias": "conceptual_physics",
|
|
"name": "mmlu_conceptual_physics",
|
|
"sample_len": 235
|
|
},
|
|
"mmlu_econometrics": {
|
|
"acc,none": 0.5614035087719298,
|
|
"acc_stderr,none": 0.04668000738510451,
|
|
"alias": "econometrics",
|
|
"name": "mmlu_econometrics",
|
|
"sample_len": 114
|
|
},
|
|
"mmlu_electrical_engineering": {
|
|
"acc,none": 0.6896551724137931,
|
|
"acc_stderr,none": 0.03855289616378948,
|
|
"alias": "electrical_engineering",
|
|
"name": "mmlu_electrical_engineering",
|
|
"sample_len": 145
|
|
},
|
|
"mmlu_elementary_mathematics": {
|
|
"acc,none": 0.36507936507936506,
|
|
"acc_stderr,none": 0.02479606060269989,
|
|
"alias": "elementary_mathematics",
|
|
"name": "mmlu_elementary_mathematics",
|
|
"sample_len": 378
|
|
},
|
|
"mmlu_formal_logic": {
|
|
"acc,none": 0.5079365079365079,
|
|
"acc_stderr,none": 0.04471572536294351,
|
|
"alias": "formal_logic",
|
|
"name": "mmlu_formal_logic",
|
|
"sample_len": 126
|
|
},
|
|
"mmlu_global_facts": {
|
|
"acc,none": 0.43,
|
|
"acc_stderr,none": 0.049756985195624305,
|
|
"alias": "global_facts",
|
|
"name": "mmlu_global_facts",
|
|
"sample_len": 100
|
|
},
|
|
"mmlu_high_school_biology": {
|
|
"acc,none": 0.8516129032258064,
|
|
"acc_stderr,none": 0.02022273755433042,
|
|
"alias": "high_school_biology",
|
|
"name": "mmlu_high_school_biology",
|
|
"sample_len": 310
|
|
},
|
|
"mmlu_high_school_chemistry": {
|
|
"acc,none": 0.6009852216748769,
|
|
"acc_stderr,none": 0.03445487686264716,
|
|
"alias": "high_school_chemistry",
|
|
"name": "mmlu_high_school_chemistry",
|
|
"sample_len": 203
|
|
},
|
|
"mmlu_high_school_computer_science": {
|
|
"acc,none": 0.78,
|
|
"acc_stderr,none": 0.041633319989322654,
|
|
"alias": "high_school_computer_science",
|
|
"name": "mmlu_high_school_computer_science",
|
|
"sample_len": 100
|
|
},
|
|
"mmlu_high_school_european_history": {
|
|
"acc,none": 0.8363636363636363,
|
|
"acc_stderr,none": 0.028887872395487978,
|
|
"alias": "high_school_european_history",
|
|
"name": "mmlu_high_school_european_history",
|
|
"sample_len": 165
|
|
},
|
|
"mmlu_high_school_geography": {
|
|
"acc,none": 0.8434343434343434,
|
|
"acc_stderr,none": 0.025890520358141458,
|
|
"alias": "high_school_geography",
|
|
"name": "mmlu_high_school_geography",
|
|
"sample_len": 198
|
|
},
|
|
"mmlu_high_school_government_and_politics": {
|
|
"acc,none": 0.8911917098445595,
|
|
"acc_stderr,none": 0.02247325333276875,
|
|
"alias": "high_school_government_and_politics",
|
|
"name": "mmlu_high_school_government_and_politics",
|
|
"sample_len": 193
|
|
},
|
|
"mmlu_high_school_macroeconomics": {
|
|
"acc,none": 0.7256410256410256,
|
|
"acc_stderr,none": 0.022622765767493235,
|
|
"alias": "high_school_macroeconomics",
|
|
"name": "mmlu_high_school_macroeconomics",
|
|
"sample_len": 390
|
|
},
|
|
"mmlu_high_school_mathematics": {
|
|
"acc,none": 0.21851851851851853,
|
|
"acc_stderr,none": 0.025195752251823727,
|
|
"alias": "high_school_mathematics",
|
|
"name": "mmlu_high_school_mathematics",
|
|
"sample_len": 270
|
|
},
|
|
"mmlu_high_school_microeconomics": {
|
|
"acc,none": 0.8445378151260504,
|
|
"acc_stderr,none": 0.023536818625398935,
|
|
"alias": "high_school_microeconomics",
|
|
"name": "mmlu_high_school_microeconomics",
|
|
"sample_len": 238
|
|
},
|
|
"mmlu_high_school_physics": {
|
|
"acc,none": 0.4768211920529801,
|
|
"acc_stderr,none": 0.04078093859163086,
|
|
"alias": "high_school_physics",
|
|
"name": "mmlu_high_school_physics",
|
|
"sample_len": 151
|
|
},
|
|
"mmlu_high_school_psychology": {
|
|
"acc,none": 0.8880733944954129,
|
|
"acc_stderr,none": 0.013517352714958736,
|
|
"alias": "high_school_psychology",
|
|
"name": "mmlu_high_school_psychology",
|
|
"sample_len": 545
|
|
},
|
|
"mmlu_high_school_statistics": {
|
|
"acc,none": 0.49537037037037035,
|
|
"acc_stderr,none": 0.034098255191635805,
|
|
"alias": "high_school_statistics",
|
|
"name": "mmlu_high_school_statistics",
|
|
"sample_len": 216
|
|
},
|
|
"mmlu_high_school_us_history": {
|
|
"acc,none": 0.8529411764705882,
|
|
"acc_stderr,none": 0.024857478080250486,
|
|
"alias": "high_school_us_history",
|
|
"name": "mmlu_high_school_us_history",
|
|
"sample_len": 204
|
|
},
|
|
"mmlu_high_school_world_history": {
|
|
"acc,none": 0.8481012658227848,
|
|
"acc_stderr,none": 0.023363878096632425,
|
|
"alias": "high_school_world_history",
|
|
"name": "mmlu_high_school_world_history",
|
|
"sample_len": 237
|
|
},
|
|
"mmlu_human_aging": {
|
|
"acc,none": 0.6860986547085202,
|
|
"acc_stderr,none": 0.031146796482972486,
|
|
"alias": "human_aging",
|
|
"name": "mmlu_human_aging",
|
|
"sample_len": 223
|
|
},
|
|
"mmlu_human_sexuality": {
|
|
"acc,none": 0.7480916030534351,
|
|
"acc_stderr,none": 0.03807387116306089,
|
|
"alias": "human_sexuality",
|
|
"name": "mmlu_human_sexuality",
|
|
"sample_len": 131
|
|
},
|
|
"mmlu_humanities": {
|
|
"acc,none": 0.5883103081827843,
|
|
"acc_stderr,none": 0.006616678303159137,
|
|
"alias": "humanities",
|
|
"name": "mmlu_humanities",
|
|
"sample_count": {
|
|
"acc,none": 4705
|
|
},
|
|
"sample_len": 4705
|
|
},
|
|
"mmlu_international_law": {
|
|
"acc,none": 0.7933884297520661,
|
|
"acc_stderr,none": 0.03695980128098826,
|
|
"alias": "international_law",
|
|
"name": "mmlu_international_law",
|
|
"sample_len": 121
|
|
},
|
|
"mmlu_jurisprudence": {
|
|
"acc,none": 0.7777777777777778,
|
|
"acc_stderr,none": 0.04019107472557348,
|
|
"alias": "jurisprudence",
|
|
"name": "mmlu_jurisprudence",
|
|
"sample_len": 108
|
|
},
|
|
"mmlu_logical_fallacies": {
|
|
"acc,none": 0.7975460122699386,
|
|
"acc_stderr,none": 0.031570650789119054,
|
|
"alias": "logical_fallacies",
|
|
"name": "mmlu_logical_fallacies",
|
|
"sample_len": 163
|
|
},
|
|
"mmlu_machine_learning": {
|
|
"acc,none": 0.5535714285714286,
|
|
"acc_stderr,none": 0.04718471485219583,
|
|
"alias": "machine_learning",
|
|
"name": "mmlu_machine_learning",
|
|
"sample_len": 112
|
|
},
|
|
"mmlu_management": {
|
|
"acc,none": 0.8446601941747572,
|
|
"acc_stderr,none": 0.03586594738573976,
|
|
"alias": "management",
|
|
"name": "mmlu_management",
|
|
"sample_len": 103
|
|
},
|
|
"mmlu_marketing": {
|
|
"acc,none": 0.8888888888888888,
|
|
"acc_stderr,none": 0.02058849131609242,
|
|
"alias": "marketing",
|
|
"name": "mmlu_marketing",
|
|
"sample_len": 234
|
|
},
|
|
"mmlu_medical_genetics": {
|
|
"acc,none": 0.83,
|
|
"acc_stderr,none": 0.03775251680686369,
|
|
"alias": "medical_genetics",
|
|
"name": "mmlu_medical_genetics",
|
|
"sample_len": 100
|
|
},
|
|
"mmlu_miscellaneous": {
|
|
"acc,none": 0.8071519795657727,
|
|
"acc_stderr,none": 0.014108533515757481,
|
|
"alias": "miscellaneous",
|
|
"name": "mmlu_miscellaneous",
|
|
"sample_len": 783
|
|
},
|
|
"mmlu_moral_disputes": {
|
|
"acc,none": 0.7167630057803468,
|
|
"acc_stderr,none": 0.024257901705323326,
|
|
"alias": "moral_disputes",
|
|
"name": "mmlu_moral_disputes",
|
|
"sample_len": 346
|
|
},
|
|
"mmlu_moral_scenarios": {
|
|
"acc,none": 0.31508379888268156,
|
|
"acc_stderr,none": 0.015536850852473702,
|
|
"alias": "moral_scenarios",
|
|
"name": "mmlu_moral_scenarios",
|
|
"sample_len": 895
|
|
},
|
|
"mmlu_nutrition": {
|
|
"acc,none": 0.7810457516339869,
|
|
"acc_stderr,none": 0.02367908986180772,
|
|
"alias": "nutrition",
|
|
"name": "mmlu_nutrition",
|
|
"sample_len": 306
|
|
},
|
|
"mmlu_other": {
|
|
"acc,none": 0.7219182491149019,
|
|
"acc_stderr,none": 0.007740746200087848,
|
|
"alias": "other",
|
|
"name": "mmlu_other",
|
|
"sample_count": {
|
|
"acc,none": 3107
|
|
},
|
|
"sample_len": 3107
|
|
},
|
|
"mmlu_philosophy": {
|
|
"acc,none": 0.7106109324758842,
|
|
"acc_stderr,none": 0.02575586592263286,
|
|
"alias": "philosophy",
|
|
"name": "mmlu_philosophy",
|
|
"sample_len": 311
|
|
},
|
|
"mmlu_prehistory": {
|
|
"acc,none": 0.7716049382716049,
|
|
"acc_stderr,none": 0.02335821184062624,
|
|
"alias": "prehistory",
|
|
"name": "mmlu_prehistory",
|
|
"sample_len": 324
|
|
},
|
|
"mmlu_professional_accounting": {
|
|
"acc,none": 0.49645390070921985,
|
|
"acc_stderr,none": 0.029826749153280853,
|
|
"alias": "professional_accounting",
|
|
"name": "mmlu_professional_accounting",
|
|
"sample_len": 282
|
|
},
|
|
"mmlu_professional_law": {
|
|
"acc,none": 0.4810951760104302,
|
|
"acc_stderr,none": 0.012761104871472521,
|
|
"alias": "professional_law",
|
|
"name": "mmlu_professional_law",
|
|
"sample_len": 1534
|
|
},
|
|
"mmlu_professional_medicine": {
|
|
"acc,none": 0.6764705882352942,
|
|
"acc_stderr,none": 0.028418208619406728,
|
|
"alias": "professional_medicine",
|
|
"name": "mmlu_professional_medicine",
|
|
"sample_len": 272
|
|
},
|
|
"mmlu_professional_psychology": {
|
|
"acc,none": 0.7336601307189542,
|
|
"acc_stderr,none": 0.017883188134667258,
|
|
"alias": "professional_psychology",
|
|
"name": "mmlu_professional_psychology",
|
|
"sample_len": 612
|
|
},
|
|
"mmlu_public_relations": {
|
|
"acc,none": 0.6545454545454545,
|
|
"acc_stderr,none": 0.045546196175410524,
|
|
"alias": "public_relations",
|
|
"name": "mmlu_public_relations",
|
|
"sample_len": 110
|
|
},
|
|
"mmlu_security_studies": {
|
|
"acc,none": 0.7673469387755102,
|
|
"acc_stderr,none": 0.02704925791589615,
|
|
"alias": "security_studies",
|
|
"name": "mmlu_security_studies",
|
|
"sample_len": 245
|
|
},
|
|
"mmlu_social_sciences": {
|
|
"acc,none": 0.7897302567435814,
|
|
"acc_stderr,none": 0.007206549343813523,
|
|
"alias": "social sciences",
|
|
"name": "mmlu_social_sciences",
|
|
"sample_count": {
|
|
"acc,none": 3077
|
|
},
|
|
"sample_len": 3077
|
|
},
|
|
"mmlu_sociology": {
|
|
"acc,none": 0.8507462686567164,
|
|
"acc_stderr,none": 0.02519692987482706,
|
|
"alias": "sociology",
|
|
"name": "mmlu_sociology",
|
|
"sample_len": 201
|
|
},
|
|
"mmlu_stem": {
|
|
"acc,none": 0.5686647637170948,
|
|
"acc_stderr,none": 0.007952947130384178,
|
|
"alias": "stem",
|
|
"name": "mmlu_stem",
|
|
"sample_count": {
|
|
"acc,none": 3153
|
|
},
|
|
"sample_len": 3153
|
|
},
|
|
"mmlu_us_foreign_policy": {
|
|
"acc,none": 0.81,
|
|
"acc_stderr,none": 0.039427724440366255,
|
|
"alias": "us_foreign_policy",
|
|
"name": "mmlu_us_foreign_policy",
|
|
"sample_len": 100
|
|
},
|
|
"mmlu_virology": {
|
|
"acc,none": 0.5120481927710844,
|
|
"acc_stderr,none": 0.038913644958358196,
|
|
"alias": "virology",
|
|
"name": "mmlu_virology",
|
|
"sample_len": 166
|
|
},
|
|
"mmlu_world_religions": {
|
|
"acc,none": 0.8304093567251462,
|
|
"acc_stderr,none": 0.02878210810540169,
|
|
"alias": "world_religions",
|
|
"name": "mmlu_world_religions",
|
|
"sample_len": 171
|
|
}
|
|
},
|
|
"system_instruction": null,
|
|
"system_instruction_sha": null,
|
|
"task_hashes": {
|
|
"mmlu_abstract_algebra": "1ecc29cafeff483bf80347adc2878dae00d01596988032f438afce6291c8cfe7",
|
|
"mmlu_anatomy": "51232e78f32967e0dd431bab9309f69be490f9998b4ddb93b641b09ba1f613cf",
|
|
"mmlu_astronomy": "3e8d7c33bd0431518a3d5d4b1666bdabccf46c4cc4cfe95a3886cb2266743f86",
|
|
"mmlu_business_ethics": "6ec93578c18c7bba58a384ecaa7e6e84c5e681262b34b1caa76abbbb0f9ebb02",
|
|
"mmlu_clinical_knowledge": "cf39fec90aa76a379e440c1f0d5f5b04d9f32edc4004ffcfb8eadcc25c18b118",
|
|
"mmlu_college_biology": "5378a297efa1cd71344228c95dcb29ba54757c7e527261400a4d65fd8ceed92b",
|
|
"mmlu_college_chemistry": "714e60ea7e340f0344ffde41b247a19b68e6384152a1bdc9acc0ecf24cd0e7cd",
|
|
"mmlu_college_computer_science": "2b66d244f98e61e6a78e5ed48b728e0ca2ca81c7757ccdd086fe6b0c1cf02da1",
|
|
"mmlu_college_mathematics": "90b1ce25d7d38abd925181ae3aad095cfd783375fe8de0607a57420f47454f0f",
|
|
"mmlu_college_medicine": "00cc41d26670913bdee6181e5b602726dde9f03e6a43236c2bf57486149845be",
|
|
"mmlu_college_physics": "2abddb12a247904f9ba60b39766b19a8967c24fe8eee021022a353b93d73742c",
|
|
"mmlu_computer_security": "8ffba21cebb9e39d5794c848a22e7e37cddff5c59f7b47a1d89872d9e8704b0c",
|
|
"mmlu_conceptual_physics": "c0da3219fe7e5292dad2aa7e72790e3eba951711007cc57618b0850d941b85ec",
|
|
"mmlu_econometrics": "bc667156b739500e02f7c8cb8f9621b6d62ea2011e1eaa3404cb45f6d16778ec",
|
|
"mmlu_electrical_engineering": "0a02cb16a31ec7debba4190ab1cc31afadb9416764e0b074b9b96553f346b892",
|
|
"mmlu_elementary_mathematics": "6048f1506468e757010d537a0ceedaadc587c18159a03e7664b5994843afedc2",
|
|
"mmlu_formal_logic": "5fef452fe37d3c5aefce6276a26a839e5ee5d655d72adf50bbf0a8ae0e8d74e4",
|
|
"mmlu_global_facts": "6163d6678b902c8cb5cd6c50f59bc0f31a9a356e977fcd6184614f48e3320e7a",
|
|
"mmlu_high_school_biology": "2aabe83bb100a352d24a0136c24e612b68490bbe818c33f06cc6cee5379ecad8",
|
|
"mmlu_high_school_chemistry": "dd3321fd42c8717094f57dd0eaeab0002d9ed192518a7de7aaef4913796c30f0",
|
|
"mmlu_high_school_computer_science": "888aabb9a7ac77299e8b2b8429fa44c1fbbb5ac8165f556a0b00b347e51de3b5",
|
|
"mmlu_high_school_european_history": "0b7be1963c7e224bffc18d72fe277408c4980f854a5fc94805fc008133640f44",
|
|
"mmlu_high_school_geography": "915db5f9f9c584c6886ae92cfc28ad9c364bf685f22c15009ad8bf064b4a8027",
|
|
"mmlu_high_school_government_and_politics": "411c6c6c179d1e2a252fab88f33123721540fc70f8223310e1f1349ec0b67753",
|
|
"mmlu_high_school_macroeconomics": "291b29471838255d1a2dd704960610e98b7e3f2fd001e3aabc6b25be0c236e29",
|
|
"mmlu_high_school_mathematics": "9b9567fc5c7875065ac91eac69621904f201cab822621440a2442a390f62b497",
|
|
"mmlu_high_school_microeconomics": "6f4a887cb73beee5414248002f75216c16d7c8ff739b37a2607c6bcfad16712b",
|
|
"mmlu_high_school_physics": "53edb93aefb6fc54fba283278bae10a58a7902902ff55edc30cb9c81745dbaa8",
|
|
"mmlu_high_school_psychology": "ce01f01ff9a3bdc3e956ada7ec7c02f13771c386d076323056ac00eec0095785",
|
|
"mmlu_high_school_statistics": "e0d652bf0cb6f927b0fc738e58a141f0ba8f9073615f4ddd70bf7a3b97998f21",
|
|
"mmlu_high_school_us_history": "1b31fdc7e0d51803fa6107a0c7aa3b6968031bc449ed3513a7349956fa0b53d3",
|
|
"mmlu_high_school_world_history": "9e352460adb245947aaf9671f135ae6484e44a115312955c45c7caaa0688f160",
|
|
"mmlu_human_aging": "4b4276095e96dc6e2872517b2becc2ebe9fad0f42b321486f6f2ceaef81f385e",
|
|
"mmlu_human_sexuality": "9395014a475c63632383587439df5c48c5dd3333eaa8d10ef9e0eaabd0997c3a",
|
|
"mmlu_international_law": "3d2036ffe61014fa8a2de679c6bee2c0c3ade112f03cd55e4a6fba5bdc56bf7b",
|
|
"mmlu_jurisprudence": "1548c06ef17823048b473d028f4889b3cf014beac4ac808521898ee141a34d23",
|
|
"mmlu_logical_fallacies": "66f9691122f3fa813b86fda1131037c03cf42babe650c7374924f41882716332",
|
|
"mmlu_machine_learning": "13ad1d3650a26d655a2a2eaa3be1f106d990e5b693b0e9006f8ebef221d2ed82",
|
|
"mmlu_management": "84a18a4a3039f23b7dd18838c89a80171025a1a45eda3305464a3cfcd8257b69",
|
|
"mmlu_marketing": "d6382d405e202221a78087356861723acb445fbd6f1db94152201f788b5c0606",
|
|
"mmlu_medical_genetics": "b4546cc241c5811dddfec47d0bba37122fbd65545c93d60f50b993e810d37e2d",
|
|
"mmlu_miscellaneous": "ecf4c3e0dc895ca5b7803d1b3f01fe3027b461f900ac001d36f5dd750c8eca1d",
|
|
"mmlu_moral_disputes": "84f580166c00063790619588b4f9fcd2f800a6a04778f08c1bada043bfdc6ec0",
|
|
"mmlu_moral_scenarios": "0dda4f43fd25e66311e64f4590698bfcc4c00c24ac58dbfec74b613512280ffc",
|
|
"mmlu_nutrition": "732959f39ac0c6c0e9c3823082d3669467531cebfb5f7550d79bcc31e5cfdbce",
|
|
"mmlu_philosophy": "ff6284cd5c6bfdb17898ef97dabc97916b70fc7485c57543edabdd44b1d8b543",
|
|
"mmlu_prehistory": "efb58ea420f5b7975ff34766ddc7470676e1ababa0b4705bfdf150f6871fdd83",
|
|
"mmlu_professional_accounting": "3c555bc35f86d097f509755f487781d2bd288de5dfe9aa59d8009a1dd0583c45",
|
|
"mmlu_professional_law": "b108c04f61cbbb4b9351de3730d7e3396710c7702c6cf75703155dbfdbc7819d",
|
|
"mmlu_professional_medicine": "0821fa48ca3715604bbef122215aea4e55f218dad6ba5191da2eb831971ac6f9",
|
|
"mmlu_professional_psychology": "ab273adb58e2b365b3811e4ed4c6485c9630f78058580273703eb2573dde9010",
|
|
"mmlu_public_relations": "da7dfa166f545d33a61c930908eee4f8d8544cad6328569b1c4b341becaf68f1",
|
|
"mmlu_security_studies": "85ea8f6c3b791998f3c1d57164b6e28bcf4498f65fc1988dfa4a6a65d31fb505",
|
|
"mmlu_sociology": "055d024e9587edc811c54e6cdd8fd74d535c6f5b1aad17cfd5f1598a1371606a",
|
|
"mmlu_us_foreign_policy": "a99e6492b6831a5df8c7f4c76edf15b500e59ab4c87dc73fdbd61d43bc704682",
|
|
"mmlu_virology": "ba4fc492ed4e9af97e30850257d2fdc5e7b53b37e7b4d68d443f7bd157aac21f",
|
|
"mmlu_world_religions": "9634e441f5f53329b4b6c760b08b1a130f8f03b3cc1902b23d07f57db76b7263"
|
|
},
|
|
"tokenizer_bos_token": [
|
|
null,
|
|
"None"
|
|
],
|
|
"tokenizer_eos_token": [
|
|
"<|im_end|>",
|
|
"151645"
|
|
],
|
|
"tokenizer_pad_token": [
|
|
"<|endoftext|>",
|
|
"151643"
|
|
],
|
|
"total_evaluation_time_seconds": "587.0024431729689",
|
|
"transformers_version": "5.11.0",
|
|
"upper_git_hash": null,
|
|
"versions": {
|
|
"mmlu": "2",
|
|
"mmlu_abstract_algebra": 1.0,
|
|
"mmlu_anatomy": 1.0,
|
|
"mmlu_astronomy": 1.0,
|
|
"mmlu_business_ethics": 1.0,
|
|
"mmlu_clinical_knowledge": 1.0,
|
|
"mmlu_college_biology": 1.0,
|
|
"mmlu_college_chemistry": 1.0,
|
|
"mmlu_college_computer_science": 1.0,
|
|
"mmlu_college_mathematics": 1.0,
|
|
"mmlu_college_medicine": 1.0,
|
|
"mmlu_college_physics": 1.0,
|
|
"mmlu_computer_security": 1.0,
|
|
"mmlu_conceptual_physics": 1.0,
|
|
"mmlu_econometrics": 1.0,
|
|
"mmlu_electrical_engineering": 1.0,
|
|
"mmlu_elementary_mathematics": 1.0,
|
|
"mmlu_formal_logic": 1.0,
|
|
"mmlu_global_facts": 1.0,
|
|
"mmlu_high_school_biology": 1.0,
|
|
"mmlu_high_school_chemistry": 1.0,
|
|
"mmlu_high_school_computer_science": 1.0,
|
|
"mmlu_high_school_european_history": 1.0,
|
|
"mmlu_high_school_geography": 1.0,
|
|
"mmlu_high_school_government_and_politics": 1.0,
|
|
"mmlu_high_school_macroeconomics": 1.0,
|
|
"mmlu_high_school_mathematics": 1.0,
|
|
"mmlu_high_school_microeconomics": 1.0,
|
|
"mmlu_high_school_physics": 1.0,
|
|
"mmlu_high_school_psychology": 1.0,
|
|
"mmlu_high_school_statistics": 1.0,
|
|
"mmlu_high_school_us_history": 1.0,
|
|
"mmlu_high_school_world_history": 1.0,
|
|
"mmlu_human_aging": 1.0,
|
|
"mmlu_human_sexuality": 1.0,
|
|
"mmlu_humanities": "2",
|
|
"mmlu_international_law": 1.0,
|
|
"mmlu_jurisprudence": 1.0,
|
|
"mmlu_logical_fallacies": 1.0,
|
|
"mmlu_machine_learning": 1.0,
|
|
"mmlu_management": 1.0,
|
|
"mmlu_marketing": 1.0,
|
|
"mmlu_medical_genetics": 1.0,
|
|
"mmlu_miscellaneous": 1.0,
|
|
"mmlu_moral_disputes": 1.0,
|
|
"mmlu_moral_scenarios": 1.0,
|
|
"mmlu_nutrition": 1.0,
|
|
"mmlu_other": "2",
|
|
"mmlu_philosophy": 1.0,
|
|
"mmlu_prehistory": 1.0,
|
|
"mmlu_professional_accounting": 1.0,
|
|
"mmlu_professional_law": 1.0,
|
|
"mmlu_professional_medicine": 1.0,
|
|
"mmlu_professional_psychology": 1.0,
|
|
"mmlu_public_relations": 1.0,
|
|
"mmlu_security_studies": 1.0,
|
|
"mmlu_social_sciences": "2",
|
|
"mmlu_sociology": 1.0,
|
|
"mmlu_stem": "2",
|
|
"mmlu_us_foreign_policy": 1.0,
|
|
"mmlu_virology": 1.0,
|
|
"mmlu_world_religions": 1.0
|
|
}
|
|
}
|