2721 lines
85 KiB
JSON
2721 lines
85 KiB
JSON
|
{
|
||
|
|
"results": {
|
||
|
|
"mmlu": {
|
||
|
|
"acc,none": 0.26100270616721266,
|
||
|
|
"acc_stderr,none": 0.0037004678245414263,
|
||
|
|
"alias": "mmlu"
|
||
|
|
},
|
||
|
|
"mmlu_humanities": {
|
||
|
|
"alias": " - humanities",
|
||
|
|
"acc,none": 0.2622741764080765,
|
||
|
|
"acc_stderr,none": 0.006405056497034186
|
||
|
|
},
|
||
|
|
"mmlu_formal_logic": {
|
||
|
|
"alias": " - formal_logic",
|
||
|
|
"acc,none": 0.16666666666666666,
|
||
|
|
"acc_stderr,none": 0.03333333333333333
|
||
|
|
},
|
||
|
|
"mmlu_high_school_european_history": {
|
||
|
|
"alias": " - high_school_european_history",
|
||
|
|
"acc,none": 0.296969696969697,
|
||
|
|
"acc_stderr,none": 0.03567969772268046
|
||
|
|
},
|
||
|
|
"mmlu_high_school_us_history": {
|
||
|
|
"alias": " - high_school_us_history",
|
||
|
|
"acc,none": 0.27450980392156865,
|
||
|
|
"acc_stderr,none": 0.03132179803083291
|
||
|
|
},
|
||
|
|
"mmlu_high_school_world_history": {
|
||
|
|
"alias": " - high_school_world_history",
|
||
|
|
"acc,none": 0.2320675105485232,
|
||
|
|
"acc_stderr,none": 0.027479744550808465
|
||
|
|
},
|
||
|
|
"mmlu_international_law": {
|
||
|
|
"alias": " - international_law",
|
||
|
|
"acc,none": 0.35537190082644626,
|
||
|
|
"acc_stderr,none": 0.043692363265739845
|
||
|
|
},
|
||
|
|
"mmlu_jurisprudence": {
|
||
|
|
"alias": " - jurisprudence",
|
||
|
|
"acc,none": 0.28703703703703703,
|
||
|
|
"acc_stderr,none": 0.04373313040914761
|
||
|
|
},
|
||
|
|
"mmlu_logical_fallacies": {
|
||
|
|
"alias": " - logical_fallacies",
|
||
|
|
"acc,none": 0.2883435582822086,
|
||
|
|
"acc_stderr,none": 0.03559039531617345
|
||
|
|
},
|
||
|
|
"mmlu_moral_disputes": {
|
||
|
|
"alias": " - moral_disputes",
|
||
|
|
"acc,none": 0.28034682080924855,
|
||
|
|
"acc_stderr,none": 0.024182427496577626
|
||
|
|
},
|
||
|
|
"mmlu_moral_scenarios": {
|
||
|
|
"alias": " - moral_scenarios",
|
||
|
|
"acc,none": 0.25251396648044694,
|
||
|
|
"acc_stderr,none": 0.014530330201468655
|
||
|
|
},
|
||
|
|
"mmlu_philosophy": {
|
||
|
|
"alias": " - philosophy",
|
||
|
|
"acc,none": 0.3247588424437299,
|
||
|
|
"acc_stderr,none": 0.02659678228769707
|
||
|
|
},
|
||
|
|
"mmlu_prehistory": {
|
||
|
|
"alias": " - prehistory",
|
||
|
|
"acc,none": 0.25,
|
||
|
|
"acc_stderr,none": 0.02409347123262133
|
||
|
|
},
|
||
|
|
"mmlu_professional_law": {
|
||
|
|
"alias": " - professional_law",
|
||
|
|
"acc,none": 0.2470664928292047,
|
||
|
|
"acc_stderr,none": 0.011015752255279445
|
||
|
|
},
|
||
|
|
"mmlu_world_religions": {
|
||
|
|
"alias": " - world_religions",
|
||
|
|
"acc,none": 0.2807017543859649,
|
||
|
|
"acc_stderr,none": 0.03446296217088429
|
||
|
|
},
|
||
|
|
"mmlu_other": {
|
||
|
|
"alias": " - other",
|
||
|
|
"acc,none": 0.2671387190215642,
|
||
|
|
"acc_stderr,none": 0.007937843638596481
|
||
|
|
},
|
||
|
|
"mmlu_business_ethics": {
|
||
|
|
"alias": " - business_ethics",
|
||
|
|
"acc,none": 0.23,
|
||
|
|
"acc_stderr,none": 0.04229525846816507
|
||
|
|
},
|
||
|
|
"mmlu_clinical_knowledge": {
|
||
|
|
"alias": " - clinical_knowledge",
|
||
|
|
"acc,none": 0.23773584905660378,
|
||
|
|
"acc_stderr,none": 0.02619980880756196
|
||
|
|
},
|
||
|
|
"mmlu_college_medicine": {
|
||
|
|
"alias": " - college_medicine",
|
||
|
|
"acc,none": 0.2543352601156069,
|
||
|
|
"acc_stderr,none": 0.03320556443085566
|
||
|
|
},
|
||
|
|
"mmlu_global_facts": {
|
||
|
|
"alias": " - global_facts",
|
||
|
|
"acc,none": 0.31,
|
||
|
|
"acc_stderr,none": 0.04648231987117317
|
||
|
|
},
|
||
|
|
"mmlu_human_aging": {
|
||
|
|
"alias": " - human_aging",
|
||
|
|
"acc,none": 0.30493273542600896,
|
||
|
|
"acc_stderr,none": 0.03089861088247754
|
||
|
|
},
|
||
|
|
"mmlu_management": {
|
||
|
|
"alias": " - management",
|
||
|
|
"acc,none": 0.24271844660194175,
|
||
|
|
"acc_stderr,none": 0.04245022486384496
|
||
|
|
},
|
||
|
|
"mmlu_marketing": {
|
||
|
|
"alias": " - marketing",
|
||
|
|
"acc,none": 0.2777777777777778,
|
||
|
|
"acc_stderr,none": 0.029343114798094483
|
||
|
|
},
|
||
|
|
"mmlu_medical_genetics": {
|
||
|
|
"alias": " - medical_genetics",
|
||
|
|
"acc,none": 0.33,
|
||
|
|
"acc_stderr,none": 0.04725815626252609
|
||
|
|
},
|
||
|
|
"mmlu_miscellaneous": {
|
||
|
|
"alias": " - miscellaneous",
|
||
|
|
"acc,none": 0.2771392081736909,
|
||
|
|
"acc_stderr,none": 0.01600563629412252
|
||
|
|
},
|
||
|
|
"mmlu_nutrition": {
|
||
|
|
"alias": " - nutrition",
|
||
|
|
"acc,none": 0.24509803921568626,
|
||
|
|
"acc_stderr,none": 0.024630048979824716
|
||
|
|
},
|
||
|
|
"mmlu_professional_accounting": {
|
||
|
|
"alias": " - professional_accounting",
|
||
|
|
"acc,none": 0.2695035460992908,
|
||
|
|
"acc_stderr,none": 0.02646903681859067
|
||
|
|
},
|
||
|
|
"mmlu_professional_medicine": {
|
||
|
|
"alias": " - professional_medicine",
|
||
|
|
"acc,none": 0.21691176470588236,
|
||
|
|
"acc_stderr,none": 0.025035845227711254
|
||
|
|
},
|
||
|
|
"mmlu_virology": {
|
||
|
|
"alias": " - virology",
|
||
|
|
"acc,none": 0.3072289156626506,
|
||
|
|
"acc_stderr,none": 0.03591566797824665
|
||
|
|
},
|
||
|
|
"mmlu_social_sciences": {
|
||
|
|
"alias": " - social_sciences",
|
||
|
|
"acc,none": 0.23431914202144946,
|
||
|
|
"acc_stderr,none": 0.007631090779162625
|
||
|
|
},
|
||
|
|
"mmlu_econometrics": {
|
||
|
|
"alias": " - econometrics",
|
||
|
|
"acc,none": 0.24561403508771928,
|
||
|
|
"acc_stderr,none": 0.040493392977481425
|
||
|
|
},
|
||
|
|
"mmlu_high_school_geography": {
|
||
|
|
"alias": " - high_school_geography",
|
||
|
|
"acc,none": 0.25252525252525254,
|
||
|
|
"acc_stderr,none": 0.03095405547036587
|
||
|
|
},
|
||
|
|
"mmlu_high_school_government_and_politics": {
|
||
|
|
"alias": " - high_school_government_and_politics",
|
||
|
|
"acc,none": 0.22279792746113988,
|
||
|
|
"acc_stderr,none": 0.030031147977641528
|
||
|
|
},
|
||
|
|
"mmlu_high_school_macroeconomics": {
|
||
|
|
"alias": " - high_school_macroeconomics",
|
||
|
|
"acc,none": 0.24615384615384617,
|
||
|
|
"acc_stderr,none": 0.02184086699042312
|
||
|
|
},
|
||
|
|
"mmlu_high_school_microeconomics": {
|
||
|
|
"alias": " - high_school_microeconomics",
|
||
|
|
"acc,none": 0.20588235294117646,
|
||
|
|
"acc_stderr,none": 0.026265024608275907
|
||
|
|
},
|
||
|
|
"mmlu_high_school_psychology": {
|
||
|
|
"alias": " - high_school_psychology",
|
||
|
|
"acc,none": 0.24036697247706423,
|
||
|
|
"acc_stderr,none": 0.01832060732096402
|
||
|
|
},
|
||
|
|
"mmlu_human_sexuality": {
|
||
|
|
"alias": " - human_sexuality",
|
||
|
|
"acc,none": 0.19083969465648856,
|
||
|
|
"acc_stderr,none": 0.03446513350752599
|
||
|
|
},
|
||
|
|
"mmlu_professional_psychology": {
|
||
|
|
"alias": " - professional_psychology",
|
||
|
|
"acc,none": 0.2647058823529412,
|
||
|
|
"acc_stderr,none": 0.01784808957491327
|
||
|
|
},
|
||
|
|
"mmlu_public_relations": {
|
||
|
|
"alias": " - public_relations",
|
||
|
|
"acc,none": 0.2909090909090909,
|
||
|
|
"acc_stderr,none": 0.04350271442923247
|
||
|
|
},
|
||
|
|
"mmlu_security_studies": {
|
||
|
|
"alias": " - security_studies",
|
||
|
|
"acc,none": 0.1673469387755102,
|
||
|
|
"acc_stderr,none": 0.023897144768914517
|
||
|
|
},
|
||
|
|
"mmlu_sociology": {
|
||
|
|
"alias": " - sociology",
|
||
|
|
"acc,none": 0.22388059701492538,
|
||
|
|
"acc_stderr,none": 0.029475250236017162
|
||
|
|
},
|
||
|
|
"mmlu_us_foreign_policy": {
|
||
|
|
"alias": " - us_foreign_policy",
|
||
|
|
"acc,none": 0.19,
|
||
|
|
"acc_stderr,none": 0.039427724440366255
|
||
|
|
},
|
||
|
|
"mmlu_stem": {
|
||
|
|
"alias": " - stem",
|
||
|
|
"acc,none": 0.2790992705359975,
|
||
|
|
"acc_stderr,none": 0.00797494423429467
|
||
|
|
},
|
||
|
|
"mmlu_abstract_algebra": {
|
||
|
|
"alias": " - abstract_algebra",
|
||
|
|
"acc,none": 0.25,
|
||
|
|
"acc_stderr,none": 0.04351941398892446
|
||
|
|
},
|
||
|
|
"mmlu_anatomy": {
|
||
|
|
"alias": " - anatomy",
|
||
|
|
"acc,none": 0.34814814814814815,
|
||
|
|
"acc_stderr,none": 0.04115324610336956
|
||
|
|
},
|
||
|
|
"mmlu_astronomy": {
|
||
|
|
"alias": " - astronomy",
|
||
|
|
"acc,none": 0.3355263157894737,
|
||
|
|
"acc_stderr,none": 0.03842498559395268
|
||
|
|
},
|
||
|
|
"mmlu_college_biology": {
|
||
|
|
"alias": " - college_biology",
|
||
|
|
"acc,none": 0.2847222222222222,
|
||
|
|
"acc_stderr,none": 0.037738099906869334
|
||
|
|
},
|
||
|
|
"mmlu_college_chemistry": {
|
||
|
|
"alias": " - college_chemistry",
|
||
|
|
"acc,none": 0.17,
|
||
|
|
"acc_stderr,none": 0.03775251680686369
|
||
|
|
},
|
||
|
|
"mmlu_college_computer_science": {
|
||
|
|
"alias": " - college_computer_science",
|
||
|
|
"acc,none": 0.33,
|
||
|
|
"acc_stderr,none": 0.04725815626252609
|
||
|
|
},
|
||
|
|
"mmlu_college_mathematics": {
|
||
|
|
"alias": " - college_mathematics",
|
||
|
|
"acc,none": 0.32,
|
||
|
|
"acc_stderr,none": 0.04688261722621507
|
||
|
|
},
|
||
|
|
"mmlu_college_physics": {
|
||
|
|
"alias": " - college_physics",
|
||
|
|
"acc,none": 0.2549019607843137,
|
||
|
|
"acc_stderr,none": 0.0433643270799318
|
||
|
|
},
|
||
|
|
"mmlu_computer_security": {
|
||
|
|
"alias": " - computer_security",
|
||
|
|
"acc,none": 0.3,
|
||
|
|
"acc_stderr,none": 0.04605661864718382
|
||
|
|
},
|
||
|
|
"mmlu_conceptual_physics": {
|
||
|
|
"alias": " - conceptual_physics",
|
||
|
|
"acc,none": 0.2170212765957447,
|
||
|
|
"acc_stderr,none": 0.026947483121496238
|
||
|
|
},
|
||
|
|
"mmlu_electrical_engineering": {
|
||
|
|
"alias": " - electrical_engineering",
|
||
|
|
"acc,none": 0.2896551724137931,
|
||
|
|
"acc_stderr,none": 0.03780019230438011
|
||
|
|
},
|
||
|
|
"mmlu_elementary_mathematics": {
|
||
|
|
"alias": " - elementary_mathematics",
|
||
|
|
"acc,none": 0.2671957671957672,
|
||
|
|
"acc_stderr,none": 0.02278967314577664
|
||
|
|
},
|
||
|
|
"mmlu_high_school_biology": {
|
||
|
|
"alias": " - high_school_biology",
|
||
|
|
"acc,none": 0.2645161290322581,
|
||
|
|
"acc_stderr,none": 0.02509189237885932
|
||
|
|
},
|
||
|
|
"mmlu_high_school_chemistry": {
|
||
|
|
"alias": " - high_school_chemistry",
|
||
|
|
"acc,none": 0.23645320197044334,
|
||
|
|
"acc_stderr,none": 0.029896114291733576
|
||
|
|
},
|
||
|
|
"mmlu_high_school_computer_science": {
|
||
|
|
"alias": " - high_school_computer_science",
|
||
|
|
"acc,none": 0.33,
|
||
|
|
"acc_stderr,none": 0.04725815626252609
|
||
|
|
},
|
||
|
|
"mmlu_high_school_mathematics": {
|
||
|
|
"alias": " - high_school_mathematics",
|
||
|
|
"acc,none": 0.25925925925925924,
|
||
|
|
"acc_stderr,none": 0.02671924078371222
|
||
|
|
},
|
||
|
|
"mmlu_high_school_physics": {
|
||
|
|
"alias": " - high_school_physics",
|
||
|
|
"acc,none": 0.271523178807947,
|
||
|
|
"acc_stderr,none": 0.03631329803969657
|
||
|
|
},
|
||
|
|
"mmlu_high_school_statistics": {
|
||
|
|
"alias": " - high_school_statistics",
|
||
|
|
"acc,none": 0.35185185185185186,
|
||
|
|
"acc_stderr,none": 0.032568505702936484
|
||
|
|
},
|
||
|
|
"mmlu_machine_learning": {
|
||
|
|
"alias": " - machine_learning",
|
||
|
|
"acc,none": 0.30357142857142855,
|
||
|
|
"acc_stderr,none": 0.043642261558410424
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"groups": {
|
||
|
|
"mmlu": {
|
||
|
|
"acc,none": 0.26100270616721266,
|
||
|
|
"acc_stderr,none": 0.0037004678245414263,
|
||
|
|
"alias": "mmlu"
|
||
|
|
},
|
||
|
|
"mmlu_humanities": {
|
||
|
|
"alias": " - humanities",
|
||
|
|
"acc,none": 0.2622741764080765,
|
||
|
|
"acc_stderr,none": 0.006405056497034186
|
||
|
|
},
|
||
|
|
"mmlu_other": {
|
||
|
|
"alias": " - other",
|
||
|
|
"acc,none": 0.2671387190215642,
|
||
|
|
"acc_stderr,none": 0.007937843638596481
|
||
|
|
},
|
||
|
|
"mmlu_social_sciences": {
|
||
|
|
"alias": " - social_sciences",
|
||
|
|
"acc,none": 0.23431914202144946,
|
||
|
|
"acc_stderr,none": 0.007631090779162625
|
||
|
|
},
|
||
|
|
"mmlu_stem": {
|
||
|
|
"alias": " - stem",
|
||
|
|
"acc,none": 0.2790992705359975,
|
||
|
|
"acc_stderr,none": 0.00797494423429467
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"group_subtasks": {
|
||
|
|
"mmlu_stem": [
|
||
|
|
"mmlu_high_school_mathematics",
|
||
|
|
"mmlu_high_school_chemistry",
|
||
|
|
"mmlu_astronomy",
|
||
|
|
"mmlu_conceptual_physics",
|
||
|
|
"mmlu_abstract_algebra",
|
||
|
|
"mmlu_college_biology",
|
||
|
|
"mmlu_machine_learning",
|
||
|
|
"mmlu_college_physics",
|
||
|
|
"mmlu_college_mathematics",
|
||
|
|
"mmlu_computer_security",
|
||
|
|
"mmlu_electrical_engineering",
|
||
|
|
"mmlu_college_chemistry",
|
||
|
|
"mmlu_elementary_mathematics",
|
||
|
|
"mmlu_high_school_statistics",
|
||
|
|
"mmlu_high_school_biology",
|
||
|
|
"mmlu_anatomy",
|
||
|
|
"mmlu_high_school_physics",
|
||
|
|
"mmlu_high_school_computer_science",
|
||
|
|
"mmlu_college_computer_science"
|
||
|
|
],
|
||
|
|
"mmlu_other": [
|
||
|
|
"mmlu_management",
|
||
|
|
"mmlu_professional_accounting",
|
||
|
|
"mmlu_college_medicine",
|
||
|
|
"mmlu_clinical_knowledge",
|
||
|
|
"mmlu_virology",
|
||
|
|
"mmlu_marketing",
|
||
|
|
"mmlu_nutrition",
|
||
|
|
"mmlu_medical_genetics",
|
||
|
|
"mmlu_global_facts",
|
||
|
|
"mmlu_miscellaneous",
|
||
|
|
"mmlu_professional_medicine",
|
||
|
|
"mmlu_human_aging",
|
||
|
|
"mmlu_business_ethics"
|
||
|
|
],
|
||
|
|
"mmlu_social_sciences": [
|
||
|
|
"mmlu_professional_psychology",
|
||
|
|
"mmlu_high_school_psychology",
|
||
|
|
"mmlu_security_studies",
|
||
|
|
"mmlu_high_school_macroeconomics",
|
||
|
|
"mmlu_us_foreign_policy",
|
||
|
|
"mmlu_high_school_government_and_politics",
|
||
|
|
"mmlu_human_sexuality",
|
||
|
|
"mmlu_econometrics",
|
||
|
|
"mmlu_public_relations",
|
||
|
|
"mmlu_sociology",
|
||
|
|
"mmlu_high_school_geography",
|
||
|
|
"mmlu_high_school_microeconomics"
|
||
|
|
],
|
||
|
|
"mmlu_humanities": [
|
||
|
|
"mmlu_international_law",
|
||
|
|
"mmlu_moral_scenarios",
|
||
|
|
"mmlu_jurisprudence",
|
||
|
|
"mmlu_professional_law",
|
||
|
|
"mmlu_high_school_world_history",
|
||
|
|
"mmlu_world_religions",
|
||
|
|
"mmlu_logical_fallacies",
|
||
|
|
"mmlu_prehistory",
|
||
|
|
"mmlu_formal_logic",
|
||
|
|
"mmlu_high_school_european_history",
|
||
|
|
"mmlu_high_school_us_history",
|
||
|
|
"mmlu_philosophy",
|
||
|
|
"mmlu_moral_disputes"
|
||
|
|
],
|
||
|
|
"mmlu": [
|
||
|
|
"mmlu_humanities",
|
||
|
|
"mmlu_social_sciences",
|
||
|
|
"mmlu_other",
|
||
|
|
"mmlu_stem"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
"configs": {
|
||
|
|
"mmlu_abstract_algebra": {
|
||
|
|
"task": "mmlu_abstract_algebra",
|
||
|
|
"task_alias": "abstract_algebra",
|
||
|
|
"group": "mmlu_stem",
|
||
|
|
"group_alias": "stem",
|
||
|
|
"dataset_path": "hails/mmlu_no_train",
|
||
|
|
"dataset_name": "abstract_algebra",
|
||
|
|
"test_split": "test",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"description": "The following are multiple choice questions (with answers) about abstract algebra.\n\n",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "first_n"
|
||
|
|
},
|
||
|
|
"num_fewshot": 5,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"metadata": {
|
||
|
|
"version": 0.0
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"mmlu_anatomy": {
|
||
|
|
"task": "mmlu_anatomy",
|
||
|
|
"task_alias": "anatomy",
|
||
|
|
"group": "mmlu_stem",
|
||
|
|
"group_alias": "stem",
|
||
|
|
"dataset_path": "hails/mmlu_no_train",
|
||
|
|
"dataset_name": "anatomy",
|
||
|
|
"test_split": "test",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"description": "The following are multiple choice questions (with answers) about anatomy.\n\n",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "first_n"
|
||
|
|
},
|
||
|
|
"num_fewshot": 5,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"metadata": {
|
||
|
|
"version": 0.0
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"mmlu_astronomy": {
|
||
|
|
"task": "mmlu_astronomy",
|
||
|
|
"task_alias": "astronomy",
|
||
|
|
"group": "mmlu_stem",
|
||
|
|
"group_alias": "stem",
|
||
|
|
"dataset_path": "hails/mmlu_no_train",
|
||
|
|
"dataset_name": "astronomy",
|
||
|
|
"test_split": "test",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"description": "The following are multiple choice questions (with answers) about astronomy.\n\n",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "first_n"
|
||
|
|
},
|
||
|
|
"num_fewshot": 5,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"metadata": {
|
||
|
|
"version": 0.0
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"mmlu_business_ethics": {
|
||
|
|
"task": "mmlu_business_ethics",
|
||
|
|
"task_alias": "business_ethics",
|
||
|
|
"group": "mmlu_other",
|
||
|
|
"group_alias": "other",
|
||
|
|
"dataset_path": "hails/mmlu_no_train",
|
||
|
|
"dataset_name": "business_ethics",
|
||
|
|
"test_split": "test",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"description": "The following are multiple choice questions (with answers) about business ethics.\n\n",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "first_n"
|
||
|
|
},
|
||
|
|
"num_fewshot": 5,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"metadata": {
|
||
|
|
"version": 0.0
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"mmlu_clinical_knowledge": {
|
||
|
|
"task": "mmlu_clinical_knowledge",
|
||
|
|
"task_alias": "clinical_knowledge",
|
||
|
|
"group": "mmlu_other",
|
||
|
|
"group_alias": "other",
|
||
|
|
"dataset_path": "hails/mmlu_no_train",
|
||
|
|
"dataset_name": "clinical_knowledge",
|
||
|
|
"test_split": "test",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"description": "The following are multiple choice questions (with answers) about clinical knowledge.\n\n",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "first_n"
|
||
|
|
},
|
||
|
|
"num_fewshot": 5,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"metadata": {
|
||
|
|
"version": 0.0
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"mmlu_college_biology": {
|
||
|
|
"task": "mmlu_college_biology",
|
||
|
|
"task_alias": "college_biology",
|
||
|
|
"group": "mmlu_stem",
|
||
|
|
"group_alias": "stem",
|
||
|
|
"dataset_path": "hails/mmlu_no_train",
|
||
|
|
"dataset_name": "college_biology",
|
||
|
|
"test_split": "test",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"description": "The following are multiple choice questions (with answers) about college biology.\n\n",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "first_n"
|
||
|
|
},
|
||
|
|
"num_fewshot": 5,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"metadata": {
|
||
|
|
"version": 0.0
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"mmlu_college_chemistry": {
|
||
|
|
"task": "mmlu_college_chemistry",
|
||
|
|
"task_alias": "college_chemistry",
|
||
|
|
"group": "mmlu_stem",
|
||
|
|
"group_alias": "stem",
|
||
|
|
"dataset_path": "hails/mmlu_no_train",
|
||
|
|
"dataset_name": "college_chemistry",
|
||
|
|
"test_split": "test",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"description": "The following are multiple choice questions (with answers) about college chemistry.\n\n",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "first_n"
|
||
|
|
},
|
||
|
|
"num_fewshot": 5,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"metadata": {
|
||
|
|
"version": 0.0
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"mmlu_college_computer_science": {
|
||
|
|
"task": "mmlu_college_computer_science",
|
||
|
|
"task_alias": "college_computer_science",
|
||
|
|
"group": "mmlu_stem",
|
||
|
|
"group_alias": "stem",
|
||
|
|
"dataset_path": "hails/mmlu_no_train",
|
||
|
|
"dataset_name": "college_computer_science",
|
||
|
|
"test_split": "test",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"description": "The following are multiple choice questions (with answers) about college computer science.\n\n",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "first_n"
|
||
|
|
},
|
||
|
|
"num_fewshot": 5,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"metadata": {
|
||
|
|
"version": 0.0
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"mmlu_college_mathematics": {
|
||
|
|
"task": "mmlu_college_mathematics",
|
||
|
|
"task_alias": "college_mathematics",
|
||
|
|
"group": "mmlu_stem",
|
||
|
|
"group_alias": "stem",
|
||
|
|
"dataset_path": "hails/mmlu_no_train",
|
||
|
|
"dataset_name": "college_mathematics",
|
||
|
|
"test_split": "test",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"description": "The following are multiple choice questions (with answers) about college mathematics.\n\n",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "first_n"
|
||
|
|
},
|
||
|
|
"num_fewshot": 5,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"metadata": {
|
||
|
|
"version": 0.0
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"mmlu_college_medicine": {
|
||
|
|
"task": "mmlu_college_medicine",
|
||
|
|
"task_alias": "college_medicine",
|
||
|
|
"group": "mmlu_other",
|
||
|
|
"group_alias": "other",
|
||
|
|
"dataset_path": "hails/mmlu_no_train",
|
||
|
|
"dataset_name": "college_medicine",
|
||
|
|
"test_split": "test",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"description": "The following are multiple choice questions (with answers) about college medicine.\n\n",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "first_n"
|
||
|
|
},
|
||
|
|
"num_fewshot": 5,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"metadata": {
|
||
|
|
"version": 0.0
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"mmlu_college_physics": {
|
||
|
|
"task": "mmlu_college_physics",
|
||
|
|
"task_alias": "college_physics",
|
||
|
|
"group": "mmlu_stem",
|
||
|
|
"group_alias": "stem",
|
||
|
|
"dataset_path": "hails/mmlu_no_train",
|
||
|
|
"dataset_name": "college_physics",
|
||
|
|
"test_split": "test",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"description": "The following are multiple choice questions (with answers) about college physics.\n\n",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "first_n"
|
||
|
|
},
|
||
|
|
"num_fewshot": 5,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"metadata": {
|
||
|
|
"version": 0.0
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"mmlu_computer_security": {
|
||
|
|
"task": "mmlu_computer_security",
|
||
|
|
"task_alias": "computer_security",
|
||
|
|
"group": "mmlu_stem",
|
||
|
|
"group_alias": "stem",
|
||
|
|
"dataset_path": "hails/mmlu_no_train",
|
||
|
|
"dataset_name": "computer_security",
|
||
|
|
"test_split": "test",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"description": "The following are multiple choice questions (with answers) about computer security.\n\n",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "first_n"
|
||
|
|
},
|
||
|
|
"num_fewshot": 5,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"metadata": {
|
||
|
|
"version": 0.0
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"mmlu_conceptual_physics": {
|
||
|
|
"task": "mmlu_conceptual_physics",
|
||
|
|
"task_alias": "conceptual_physics",
|
||
|
|
"group": "mmlu_stem",
|
||
|
|
"group_alias": "stem",
|
||
|
|
"dataset_path": "hails/mmlu_no_train",
|
||
|
|
"dataset_name": "conceptual_physics",
|
||
|
|
"test_split": "test",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"description": "The following are multiple choice questions (with answers) about conceptual physics.\n\n",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "first_n"
|
||
|
|
},
|
||
|
|
"num_fewshot": 5,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"metadata": {
|
||
|
|
"version": 0.0
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"mmlu_econometrics": {
|
||
|
|
"task": "mmlu_econometrics",
|
||
|
|
"task_alias": "econometrics",
|
||
|
|
"group": "mmlu_social_sciences",
|
||
|
|
"group_alias": "social_sciences",
|
||
|
|
"dataset_path": "hails/mmlu_no_train",
|
||
|
|
"dataset_name": "econometrics",
|
||
|
|
"test_split": "test",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"description": "The following are multiple choice questions (with answers) about econometrics.\n\n",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "first_n"
|
||
|
|
},
|
||
|
|
"num_fewshot": 5,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"metadata": {
|
||
|
|
"version": 0.0
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"mmlu_electrical_engineering": {
|
||
|
|
"task": "mmlu_electrical_engineering",
|
||
|
|
"task_alias": "electrical_engineering",
|
||
|
|
"group": "mmlu_stem",
|
||
|
|
"group_alias": "stem",
|
||
|
|
"dataset_path": "hails/mmlu_no_train",
|
||
|
|
"dataset_name": "electrical_engineering",
|
||
|
|
"test_split": "test",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"description": "The following are multiple choice questions (with answers) about electrical engineering.\n\n",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "first_n"
|
||
|
|
},
|
||
|
|
"num_fewshot": 5,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"metadata": {
|
||
|
|
"version": 0.0
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"mmlu_elementary_mathematics": {
|
||
|
|
"task": "mmlu_elementary_mathematics",
|
||
|
|
"task_alias": "elementary_mathematics",
|
||
|
|
"group": "mmlu_stem",
|
||
|
|
"group_alias": "stem",
|
||
|
|
"dataset_path": "hails/mmlu_no_train",
|
||
|
|
"dataset_name": "elementary_mathematics",
|
||
|
|
"test_split": "test",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"description": "The following are multiple choice questions (with answers) about elementary mathematics.\n\n",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "first_n"
|
||
|
|
},
|
||
|
|
"num_fewshot": 5,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"metadata": {
|
||
|
|
"version": 0.0
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"mmlu_formal_logic": {
|
||
|
|
"task": "mmlu_formal_logic",
|
||
|
|
"task_alias": "formal_logic",
|
||
|
|
"group": "mmlu_humanities",
|
||
|
|
"group_alias": "humanities",
|
||
|
|
"dataset_path": "hails/mmlu_no_train",
|
||
|
|
"dataset_name": "formal_logic",
|
||
|
|
"test_split": "test",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"description": "The following are multiple choice questions (with answers) about formal logic.\n\n",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "first_n"
|
||
|
|
},
|
||
|
|
"num_fewshot": 5,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"metadata": {
|
||
|
|
"version": 0.0
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"mmlu_global_facts": {
|
||
|
|
"task": "mmlu_global_facts",
|
||
|
|
"task_alias": "global_facts",
|
||
|
|
"group": "mmlu_other",
|
||
|
|
"group_alias": "other",
|
||
|
|
"dataset_path": "hails/mmlu_no_train",
|
||
|
|
"dataset_name": "global_facts",
|
||
|
|
"test_split": "test",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"description": "The following are multiple choice questions (with answers) about global facts.\n\n",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "first_n"
|
||
|
|
},
|
||
|
|
"num_fewshot": 5,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"metadata": {
|
||
|
|
"version": 0.0
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"mmlu_high_school_biology": {
|
||
|
|
"task": "mmlu_high_school_biology",
|
||
|
|
"task_alias": "high_school_biology",
|
||
|
|
"group": "mmlu_stem",
|
||
|
|
"group_alias": "stem",
|
||
|
|
"dataset_path": "hails/mmlu_no_train",
|
||
|
|
"dataset_name": "high_school_biology",
|
||
|
|
"test_split": "test",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"description": "The following are multiple choice questions (with answers) about high school biology.\n\n",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "first_n"
|
||
|
|
},
|
||
|
|
"num_fewshot": 5,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"metadata": {
|
||
|
|
"version": 0.0
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"mmlu_high_school_chemistry": {
|
||
|
|
"task": "mmlu_high_school_chemistry",
|
||
|
|
"task_alias": "high_school_chemistry",
|
||
|
|
"group": "mmlu_stem",
|
||
|
|
"group_alias": "stem",
|
||
|
|
"dataset_path": "hails/mmlu_no_train",
|
||
|
|
"dataset_name": "high_school_chemistry",
|
||
|
|
"test_split": "test",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"description": "The following are multiple choice questions (with answers) about high school chemistry.\n\n",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "first_n"
|
||
|
|
},
|
||
|
|
"num_fewshot": 5,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"metadata": {
|
||
|
|
"version": 0.0
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"mmlu_high_school_computer_science": {
|
||
|
|
"task": "mmlu_high_school_computer_science",
|
||
|
|
"task_alias": "high_school_computer_science",
|
||
|
|
"group": "mmlu_stem",
|
||
|
|
"group_alias": "stem",
|
||
|
|
"dataset_path": "hails/mmlu_no_train",
|
||
|
|
"dataset_name": "high_school_computer_science",
|
||
|
|
"test_split": "test",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"description": "The following are multiple choice questions (with answers) about high school computer science.\n\n",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "first_n"
|
||
|
|
},
|
||
|
|
"num_fewshot": 5,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"metadata": {
|
||
|
|
"version": 0.0
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"mmlu_high_school_european_history": {
|
||
|
|
"task": "mmlu_high_school_european_history",
|
||
|
|
"task_alias": "high_school_european_history",
|
||
|
|
"group": "mmlu_humanities",
|
||
|
|
"group_alias": "humanities",
|
||
|
|
"dataset_path": "hails/mmlu_no_train",
|
||
|
|
"dataset_name": "high_school_european_history",
|
||
|
|
"test_split": "test",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"description": "The following are multiple choice questions (with answers) about high school european history.\n\n",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "first_n"
|
||
|
|
},
|
||
|
|
"num_fewshot": 5,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"metadata": {
|
||
|
|
"version": 0.0
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"mmlu_high_school_geography": {
|
||
|
|
"task": "mmlu_high_school_geography",
|
||
|
|
"task_alias": "high_school_geography",
|
||
|
|
"group": "mmlu_social_sciences",
|
||
|
|
"group_alias": "social_sciences",
|
||
|
|
"dataset_path": "hails/mmlu_no_train",
|
||
|
|
"dataset_name": "high_school_geography",
|
||
|
|
"test_split": "test",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"description": "The following are multiple choice questions (with answers) about high school geography.\n\n",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "first_n"
|
||
|
|
},
|
||
|
|
"num_fewshot": 5,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"metadata": {
|
||
|
|
"version": 0.0
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"mmlu_high_school_government_and_politics": {
|
||
|
|
"task": "mmlu_high_school_government_and_politics",
|
||
|
|
"task_alias": "high_school_government_and_politics",
|
||
|
|
"group": "mmlu_social_sciences",
|
||
|
|
"group_alias": "social_sciences",
|
||
|
|
"dataset_path": "hails/mmlu_no_train",
|
||
|
|
"dataset_name": "high_school_government_and_politics",
|
||
|
|
"test_split": "test",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"description": "The following are multiple choice questions (with answers) about high school government and politics.\n\n",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "first_n"
|
||
|
|
},
|
||
|
|
"num_fewshot": 5,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"metadata": {
|
||
|
|
"version": 0.0
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"mmlu_high_school_macroeconomics": {
|
||
|
|
"task": "mmlu_high_school_macroeconomics",
|
||
|
|
"task_alias": "high_school_macroeconomics",
|
||
|
|
"group": "mmlu_social_sciences",
|
||
|
|
"group_alias": "social_sciences",
|
||
|
|
"dataset_path": "hails/mmlu_no_train",
|
||
|
|
"dataset_name": "high_school_macroeconomics",
|
||
|
|
"test_split": "test",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"description": "The following are multiple choice questions (with answers) about high school macroeconomics.\n\n",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "first_n"
|
||
|
|
},
|
||
|
|
"num_fewshot": 5,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"metadata": {
|
||
|
|
"version": 0.0
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"mmlu_high_school_mathematics": {
|
||
|
|
"task": "mmlu_high_school_mathematics",
|
||
|
|
"task_alias": "high_school_mathematics",
|
||
|
|
"group": "mmlu_stem",
|
||
|
|
"group_alias": "stem",
|
||
|
|
"dataset_path": "hails/mmlu_no_train",
|
||
|
|
"dataset_name": "high_school_mathematics",
|
||
|
|
"test_split": "test",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"description": "The following are multiple choice questions (with answers) about high school mathematics.\n\n",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "first_n"
|
||
|
|
},
|
||
|
|
"num_fewshot": 5,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"metadata": {
|
||
|
|
"version": 0.0
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"mmlu_high_school_microeconomics": {
|
||
|
|
"task": "mmlu_high_school_microeconomics",
|
||
|
|
"task_alias": "high_school_microeconomics",
|
||
|
|
"group": "mmlu_social_sciences",
|
||
|
|
"group_alias": "social_sciences",
|
||
|
|
"dataset_path": "hails/mmlu_no_train",
|
||
|
|
"dataset_name": "high_school_microeconomics",
|
||
|
|
"test_split": "test",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"description": "The following are multiple choice questions (with answers) about high school microeconomics.\n\n",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "first_n"
|
||
|
|
},
|
||
|
|
"num_fewshot": 5,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"metadata": {
|
||
|
|
"version": 0.0
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"mmlu_high_school_physics": {
|
||
|
|
"task": "mmlu_high_school_physics",
|
||
|
|
"task_alias": "high_school_physics",
|
||
|
|
"group": "mmlu_stem",
|
||
|
|
"group_alias": "stem",
|
||
|
|
"dataset_path": "hails/mmlu_no_train",
|
||
|
|
"dataset_name": "high_school_physics",
|
||
|
|
"test_split": "test",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"description": "The following are multiple choice questions (with answers) about high school physics.\n\n",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "first_n"
|
||
|
|
},
|
||
|
|
"num_fewshot": 5,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"metadata": {
|
||
|
|
"version": 0.0
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"mmlu_high_school_psychology": {
|
||
|
|
"task": "mmlu_high_school_psychology",
|
||
|
|
"task_alias": "high_school_psychology",
|
||
|
|
"group": "mmlu_social_sciences",
|
||
|
|
"group_alias": "social_sciences",
|
||
|
|
"dataset_path": "hails/mmlu_no_train",
|
||
|
|
"dataset_name": "high_school_psychology",
|
||
|
|
"test_split": "test",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"description": "The following are multiple choice questions (with answers) about high school psychology.\n\n",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "first_n"
|
||
|
|
},
|
||
|
|
"num_fewshot": 5,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"metadata": {
|
||
|
|
"version": 0.0
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"mmlu_high_school_statistics": {
|
||
|
|
"task": "mmlu_high_school_statistics",
|
||
|
|
"task_alias": "high_school_statistics",
|
||
|
|
"group": "mmlu_stem",
|
||
|
|
"group_alias": "stem",
|
||
|
|
"dataset_path": "hails/mmlu_no_train",
|
||
|
|
"dataset_name": "high_school_statistics",
|
||
|
|
"test_split": "test",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"description": "The following are multiple choice questions (with answers) about high school statistics.\n\n",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "first_n"
|
||
|
|
},
|
||
|
|
"num_fewshot": 5,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"metadata": {
|
||
|
|
"version": 0.0
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"mmlu_high_school_us_history": {
|
||
|
|
"task": "mmlu_high_school_us_history",
|
||
|
|
"task_alias": "high_school_us_history",
|
||
|
|
"group": "mmlu_humanities",
|
||
|
|
"group_alias": "humanities",
|
||
|
|
"dataset_path": "hails/mmlu_no_train",
|
||
|
|
"dataset_name": "high_school_us_history",
|
||
|
|
"test_split": "test",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"description": "The following are multiple choice questions (with answers) about high school us history.\n\n",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "first_n"
|
||
|
|
},
|
||
|
|
"num_fewshot": 5,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"metadata": {
|
||
|
|
"version": 0.0
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"mmlu_high_school_world_history": {
|
||
|
|
"task": "mmlu_high_school_world_history",
|
||
|
|
"task_alias": "high_school_world_history",
|
||
|
|
"group": "mmlu_humanities",
|
||
|
|
"group_alias": "humanities",
|
||
|
|
"dataset_path": "hails/mmlu_no_train",
|
||
|
|
"dataset_name": "high_school_world_history",
|
||
|
|
"test_split": "test",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"description": "The following are multiple choice questions (with answers) about high school world history.\n\n",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "first_n"
|
||
|
|
},
|
||
|
|
"num_fewshot": 5,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"metadata": {
|
||
|
|
"version": 0.0
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"mmlu_human_aging": {
|
||
|
|
"task": "mmlu_human_aging",
|
||
|
|
"task_alias": "human_aging",
|
||
|
|
"group": "mmlu_other",
|
||
|
|
"group_alias": "other",
|
||
|
|
"dataset_path": "hails/mmlu_no_train",
|
||
|
|
"dataset_name": "human_aging",
|
||
|
|
"test_split": "test",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"description": "The following are multiple choice questions (with answers) about human aging.\n\n",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "first_n"
|
||
|
|
},
|
||
|
|
"num_fewshot": 5,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"metadata": {
|
||
|
|
"version": 0.0
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"mmlu_human_sexuality": {
|
||
|
|
"task": "mmlu_human_sexuality",
|
||
|
|
"task_alias": "human_sexuality",
|
||
|
|
"group": "mmlu_social_sciences",
|
||
|
|
"group_alias": "social_sciences",
|
||
|
|
"dataset_path": "hails/mmlu_no_train",
|
||
|
|
"dataset_name": "human_sexuality",
|
||
|
|
"test_split": "test",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"description": "The following are multiple choice questions (with answers) about human sexuality.\n\n",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "first_n"
|
||
|
|
},
|
||
|
|
"num_fewshot": 5,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"metadata": {
|
||
|
|
"version": 0.0
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"mmlu_international_law": {
|
||
|
|
"task": "mmlu_international_law",
|
||
|
|
"task_alias": "international_law",
|
||
|
|
"group": "mmlu_humanities",
|
||
|
|
"group_alias": "humanities",
|
||
|
|
"dataset_path": "hails/mmlu_no_train",
|
||
|
|
"dataset_name": "international_law",
|
||
|
|
"test_split": "test",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"description": "The following are multiple choice questions (with answers) about international law.\n\n",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "first_n"
|
||
|
|
},
|
||
|
|
"num_fewshot": 5,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"metadata": {
|
||
|
|
"version": 0.0
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"mmlu_jurisprudence": {
|
||
|
|
"task": "mmlu_jurisprudence",
|
||
|
|
"task_alias": "jurisprudence",
|
||
|
|
"group": "mmlu_humanities",
|
||
|
|
"group_alias": "humanities",
|
||
|
|
"dataset_path": "hails/mmlu_no_train",
|
||
|
|
"dataset_name": "jurisprudence",
|
||
|
|
"test_split": "test",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"description": "The following are multiple choice questions (with answers) about jurisprudence.\n\n",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "first_n"
|
||
|
|
},
|
||
|
|
"num_fewshot": 5,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"metadata": {
|
||
|
|
"version": 0.0
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"mmlu_logical_fallacies": {
|
||
|
|
"task": "mmlu_logical_fallacies",
|
||
|
|
"task_alias": "logical_fallacies",
|
||
|
|
"group": "mmlu_humanities",
|
||
|
|
"group_alias": "humanities",
|
||
|
|
"dataset_path": "hails/mmlu_no_train",
|
||
|
|
"dataset_name": "logical_fallacies",
|
||
|
|
"test_split": "test",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"description": "The following are multiple choice questions (with answers) about logical fallacies.\n\n",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "first_n"
|
||
|
|
},
|
||
|
|
"num_fewshot": 5,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"metadata": {
|
||
|
|
"version": 0.0
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"mmlu_machine_learning": {
|
||
|
|
"task": "mmlu_machine_learning",
|
||
|
|
"task_alias": "machine_learning",
|
||
|
|
"group": "mmlu_stem",
|
||
|
|
"group_alias": "stem",
|
||
|
|
"dataset_path": "hails/mmlu_no_train",
|
||
|
|
"dataset_name": "machine_learning",
|
||
|
|
"test_split": "test",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"description": "The following are multiple choice questions (with answers) about machine learning.\n\n",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "first_n"
|
||
|
|
},
|
||
|
|
"num_fewshot": 5,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"metadata": {
|
||
|
|
"version": 0.0
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"mmlu_management": {
|
||
|
|
"task": "mmlu_management",
|
||
|
|
"task_alias": "management",
|
||
|
|
"group": "mmlu_other",
|
||
|
|
"group_alias": "other",
|
||
|
|
"dataset_path": "hails/mmlu_no_train",
|
||
|
|
"dataset_name": "management",
|
||
|
|
"test_split": "test",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"description": "The following are multiple choice questions (with answers) about management.\n\n",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "first_n"
|
||
|
|
},
|
||
|
|
"num_fewshot": 5,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"metadata": {
|
||
|
|
"version": 0.0
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"mmlu_marketing": {
|
||
|
|
"task": "mmlu_marketing",
|
||
|
|
"task_alias": "marketing",
|
||
|
|
"group": "mmlu_other",
|
||
|
|
"group_alias": "other",
|
||
|
|
"dataset_path": "hails/mmlu_no_train",
|
||
|
|
"dataset_name": "marketing",
|
||
|
|
"test_split": "test",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"description": "The following are multiple choice questions (with answers) about marketing.\n\n",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "first_n"
|
||
|
|
},
|
||
|
|
"num_fewshot": 5,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"metadata": {
|
||
|
|
"version": 0.0
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"mmlu_medical_genetics": {
|
||
|
|
"task": "mmlu_medical_genetics",
|
||
|
|
"task_alias": "medical_genetics",
|
||
|
|
"group": "mmlu_other",
|
||
|
|
"group_alias": "other",
|
||
|
|
"dataset_path": "hails/mmlu_no_train",
|
||
|
|
"dataset_name": "medical_genetics",
|
||
|
|
"test_split": "test",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"description": "The following are multiple choice questions (with answers) about medical genetics.\n\n",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "first_n"
|
||
|
|
},
|
||
|
|
"num_fewshot": 5,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"metadata": {
|
||
|
|
"version": 0.0
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"mmlu_miscellaneous": {
|
||
|
|
"task": "mmlu_miscellaneous",
|
||
|
|
"task_alias": "miscellaneous",
|
||
|
|
"group": "mmlu_other",
|
||
|
|
"group_alias": "other",
|
||
|
|
"dataset_path": "hails/mmlu_no_train",
|
||
|
|
"dataset_name": "miscellaneous",
|
||
|
|
"test_split": "test",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"description": "The following are multiple choice questions (with answers) about miscellaneous.\n\n",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "first_n"
|
||
|
|
},
|
||
|
|
"num_fewshot": 5,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"metadata": {
|
||
|
|
"version": 0.0
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"mmlu_moral_disputes": {
|
||
|
|
"task": "mmlu_moral_disputes",
|
||
|
|
"task_alias": "moral_disputes",
|
||
|
|
"group": "mmlu_humanities",
|
||
|
|
"group_alias": "humanities",
|
||
|
|
"dataset_path": "hails/mmlu_no_train",
|
||
|
|
"dataset_name": "moral_disputes",
|
||
|
|
"test_split": "test",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"description": "The following are multiple choice questions (with answers) about moral disputes.\n\n",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "first_n"
|
||
|
|
},
|
||
|
|
"num_fewshot": 5,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"metadata": {
|
||
|
|
"version": 0.0
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"mmlu_moral_scenarios": {
|
||
|
|
"task": "mmlu_moral_scenarios",
|
||
|
|
"task_alias": "moral_scenarios",
|
||
|
|
"group": "mmlu_humanities",
|
||
|
|
"group_alias": "humanities",
|
||
|
|
"dataset_path": "hails/mmlu_no_train",
|
||
|
|
"dataset_name": "moral_scenarios",
|
||
|
|
"test_split": "test",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"description": "The following are multiple choice questions (with answers) about moral scenarios.\n\n",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "first_n"
|
||
|
|
},
|
||
|
|
"num_fewshot": 5,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"metadata": {
|
||
|
|
"version": 0.0
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"mmlu_nutrition": {
|
||
|
|
"task": "mmlu_nutrition",
|
||
|
|
"task_alias": "nutrition",
|
||
|
|
"group": "mmlu_other",
|
||
|
|
"group_alias": "other",
|
||
|
|
"dataset_path": "hails/mmlu_no_train",
|
||
|
|
"dataset_name": "nutrition",
|
||
|
|
"test_split": "test",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"description": "The following are multiple choice questions (with answers) about nutrition.\n\n",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "first_n"
|
||
|
|
},
|
||
|
|
"num_fewshot": 5,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"metadata": {
|
||
|
|
"version": 0.0
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"mmlu_philosophy": {
|
||
|
|
"task": "mmlu_philosophy",
|
||
|
|
"task_alias": "philosophy",
|
||
|
|
"group": "mmlu_humanities",
|
||
|
|
"group_alias": "humanities",
|
||
|
|
"dataset_path": "hails/mmlu_no_train",
|
||
|
|
"dataset_name": "philosophy",
|
||
|
|
"test_split": "test",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"description": "The following are multiple choice questions (with answers) about philosophy.\n\n",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "first_n"
|
||
|
|
},
|
||
|
|
"num_fewshot": 5,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"metadata": {
|
||
|
|
"version": 0.0
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"mmlu_prehistory": {
|
||
|
|
"task": "mmlu_prehistory",
|
||
|
|
"task_alias": "prehistory",
|
||
|
|
"group": "mmlu_humanities",
|
||
|
|
"group_alias": "humanities",
|
||
|
|
"dataset_path": "hails/mmlu_no_train",
|
||
|
|
"dataset_name": "prehistory",
|
||
|
|
"test_split": "test",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"description": "The following are multiple choice questions (with answers) about prehistory.\n\n",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "first_n"
|
||
|
|
},
|
||
|
|
"num_fewshot": 5,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"metadata": {
|
||
|
|
"version": 0.0
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"mmlu_professional_accounting": {
|
||
|
|
"task": "mmlu_professional_accounting",
|
||
|
|
"task_alias": "professional_accounting",
|
||
|
|
"group": "mmlu_other",
|
||
|
|
"group_alias": "other",
|
||
|
|
"dataset_path": "hails/mmlu_no_train",
|
||
|
|
"dataset_name": "professional_accounting",
|
||
|
|
"test_split": "test",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"description": "The following are multiple choice questions (with answers) about professional accounting.\n\n",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "first_n"
|
||
|
|
},
|
||
|
|
"num_fewshot": 5,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"metadata": {
|
||
|
|
"version": 0.0
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"mmlu_professional_law": {
|
||
|
|
"task": "mmlu_professional_law",
|
||
|
|
"task_alias": "professional_law",
|
||
|
|
"group": "mmlu_humanities",
|
||
|
|
"group_alias": "humanities",
|
||
|
|
"dataset_path": "hails/mmlu_no_train",
|
||
|
|
"dataset_name": "professional_law",
|
||
|
|
"test_split": "test",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"description": "The following are multiple choice questions (with answers) about professional law.\n\n",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "first_n"
|
||
|
|
},
|
||
|
|
"num_fewshot": 5,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"metadata": {
|
||
|
|
"version": 0.0
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"mmlu_professional_medicine": {
|
||
|
|
"task": "mmlu_professional_medicine",
|
||
|
|
"task_alias": "professional_medicine",
|
||
|
|
"group": "mmlu_other",
|
||
|
|
"group_alias": "other",
|
||
|
|
"dataset_path": "hails/mmlu_no_train",
|
||
|
|
"dataset_name": "professional_medicine",
|
||
|
|
"test_split": "test",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"description": "The following are multiple choice questions (with answers) about professional medicine.\n\n",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "first_n"
|
||
|
|
},
|
||
|
|
"num_fewshot": 5,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"metadata": {
|
||
|
|
"version": 0.0
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"mmlu_professional_psychology": {
|
||
|
|
"task": "mmlu_professional_psychology",
|
||
|
|
"task_alias": "professional_psychology",
|
||
|
|
"group": "mmlu_social_sciences",
|
||
|
|
"group_alias": "social_sciences",
|
||
|
|
"dataset_path": "hails/mmlu_no_train",
|
||
|
|
"dataset_name": "professional_psychology",
|
||
|
|
"test_split": "test",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"description": "The following are multiple choice questions (with answers) about professional psychology.\n\n",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "first_n"
|
||
|
|
},
|
||
|
|
"num_fewshot": 5,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"metadata": {
|
||
|
|
"version": 0.0
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"mmlu_public_relations": {
|
||
|
|
"task": "mmlu_public_relations",
|
||
|
|
"task_alias": "public_relations",
|
||
|
|
"group": "mmlu_social_sciences",
|
||
|
|
"group_alias": "social_sciences",
|
||
|
|
"dataset_path": "hails/mmlu_no_train",
|
||
|
|
"dataset_name": "public_relations",
|
||
|
|
"test_split": "test",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"description": "The following are multiple choice questions (with answers) about public relations.\n\n",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "first_n"
|
||
|
|
},
|
||
|
|
"num_fewshot": 5,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"metadata": {
|
||
|
|
"version": 0.0
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"mmlu_security_studies": {
|
||
|
|
"task": "mmlu_security_studies",
|
||
|
|
"task_alias": "security_studies",
|
||
|
|
"group": "mmlu_social_sciences",
|
||
|
|
"group_alias": "social_sciences",
|
||
|
|
"dataset_path": "hails/mmlu_no_train",
|
||
|
|
"dataset_name": "security_studies",
|
||
|
|
"test_split": "test",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"description": "The following are multiple choice questions (with answers) about security studies.\n\n",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "first_n"
|
||
|
|
},
|
||
|
|
"num_fewshot": 5,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"metadata": {
|
||
|
|
"version": 0.0
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"mmlu_sociology": {
|
||
|
|
"task": "mmlu_sociology",
|
||
|
|
"task_alias": "sociology",
|
||
|
|
"group": "mmlu_social_sciences",
|
||
|
|
"group_alias": "social_sciences",
|
||
|
|
"dataset_path": "hails/mmlu_no_train",
|
||
|
|
"dataset_name": "sociology",
|
||
|
|
"test_split": "test",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"description": "The following are multiple choice questions (with answers) about sociology.\n\n",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "first_n"
|
||
|
|
},
|
||
|
|
"num_fewshot": 5,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"metadata": {
|
||
|
|
"version": 0.0
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"mmlu_us_foreign_policy": {
|
||
|
|
"task": "mmlu_us_foreign_policy",
|
||
|
|
"task_alias": "us_foreign_policy",
|
||
|
|
"group": "mmlu_social_sciences",
|
||
|
|
"group_alias": "social_sciences",
|
||
|
|
"dataset_path": "hails/mmlu_no_train",
|
||
|
|
"dataset_name": "us_foreign_policy",
|
||
|
|
"test_split": "test",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"description": "The following are multiple choice questions (with answers) about us foreign policy.\n\n",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "first_n"
|
||
|
|
},
|
||
|
|
"num_fewshot": 5,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"metadata": {
|
||
|
|
"version": 0.0
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"mmlu_virology": {
|
||
|
|
"task": "mmlu_virology",
|
||
|
|
"task_alias": "virology",
|
||
|
|
"group": "mmlu_other",
|
||
|
|
"group_alias": "other",
|
||
|
|
"dataset_path": "hails/mmlu_no_train",
|
||
|
|
"dataset_name": "virology",
|
||
|
|
"test_split": "test",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"description": "The following are multiple choice questions (with answers) about virology.\n\n",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "first_n"
|
||
|
|
},
|
||
|
|
"num_fewshot": 5,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"metadata": {
|
||
|
|
"version": 0.0
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"mmlu_world_religions": {
|
||
|
|
"task": "mmlu_world_religions",
|
||
|
|
"task_alias": "world_religions",
|
||
|
|
"group": "mmlu_humanities",
|
||
|
|
"group_alias": "humanities",
|
||
|
|
"dataset_path": "hails/mmlu_no_train",
|
||
|
|
"dataset_name": "world_religions",
|
||
|
|
"test_split": "test",
|
||
|
|
"fewshot_split": "dev",
|
||
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
||
|
|
"doc_to_target": "answer",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"A",
|
||
|
|
"B",
|
||
|
|
"C",
|
||
|
|
"D"
|
||
|
|
],
|
||
|
|
"description": "The following are multiple choice questions (with answers) about world religions.\n\n",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "first_n"
|
||
|
|
},
|
||
|
|
"num_fewshot": 5,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"metadata": {
|
||
|
|
"version": 0.0
|
||
|
|
}
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"versions": {
|
||
|
|
"mmlu_abstract_algebra": 0.0,
|
||
|
|
"mmlu_anatomy": 0.0,
|
||
|
|
"mmlu_astronomy": 0.0,
|
||
|
|
"mmlu_business_ethics": 0.0,
|
||
|
|
"mmlu_clinical_knowledge": 0.0,
|
||
|
|
"mmlu_college_biology": 0.0,
|
||
|
|
"mmlu_college_chemistry": 0.0,
|
||
|
|
"mmlu_college_computer_science": 0.0,
|
||
|
|
"mmlu_college_mathematics": 0.0,
|
||
|
|
"mmlu_college_medicine": 0.0,
|
||
|
|
"mmlu_college_physics": 0.0,
|
||
|
|
"mmlu_computer_security": 0.0,
|
||
|
|
"mmlu_conceptual_physics": 0.0,
|
||
|
|
"mmlu_econometrics": 0.0,
|
||
|
|
"mmlu_electrical_engineering": 0.0,
|
||
|
|
"mmlu_elementary_mathematics": 0.0,
|
||
|
|
"mmlu_formal_logic": 0.0,
|
||
|
|
"mmlu_global_facts": 0.0,
|
||
|
|
"mmlu_high_school_biology": 0.0,
|
||
|
|
"mmlu_high_school_chemistry": 0.0,
|
||
|
|
"mmlu_high_school_computer_science": 0.0,
|
||
|
|
"mmlu_high_school_european_history": 0.0,
|
||
|
|
"mmlu_high_school_geography": 0.0,
|
||
|
|
"mmlu_high_school_government_and_politics": 0.0,
|
||
|
|
"mmlu_high_school_macroeconomics": 0.0,
|
||
|
|
"mmlu_high_school_mathematics": 0.0,
|
||
|
|
"mmlu_high_school_microeconomics": 0.0,
|
||
|
|
"mmlu_high_school_physics": 0.0,
|
||
|
|
"mmlu_high_school_psychology": 0.0,
|
||
|
|
"mmlu_high_school_statistics": 0.0,
|
||
|
|
"mmlu_high_school_us_history": 0.0,
|
||
|
|
"mmlu_high_school_world_history": 0.0,
|
||
|
|
"mmlu_human_aging": 0.0,
|
||
|
|
"mmlu_human_sexuality": 0.0,
|
||
|
|
"mmlu_international_law": 0.0,
|
||
|
|
"mmlu_jurisprudence": 0.0,
|
||
|
|
"mmlu_logical_fallacies": 0.0,
|
||
|
|
"mmlu_machine_learning": 0.0,
|
||
|
|
"mmlu_management": 0.0,
|
||
|
|
"mmlu_marketing": 0.0,
|
||
|
|
"mmlu_medical_genetics": 0.0,
|
||
|
|
"mmlu_miscellaneous": 0.0,
|
||
|
|
"mmlu_moral_disputes": 0.0,
|
||
|
|
"mmlu_moral_scenarios": 0.0,
|
||
|
|
"mmlu_nutrition": 0.0,
|
||
|
|
"mmlu_philosophy": 0.0,
|
||
|
|
"mmlu_prehistory": 0.0,
|
||
|
|
"mmlu_professional_accounting": 0.0,
|
||
|
|
"mmlu_professional_law": 0.0,
|
||
|
|
"mmlu_professional_medicine": 0.0,
|
||
|
|
"mmlu_professional_psychology": 0.0,
|
||
|
|
"mmlu_public_relations": 0.0,
|
||
|
|
"mmlu_security_studies": 0.0,
|
||
|
|
"mmlu_sociology": 0.0,
|
||
|
|
"mmlu_us_foreign_policy": 0.0,
|
||
|
|
"mmlu_virology": 0.0,
|
||
|
|
"mmlu_world_religions": 0.0
|
||
|
|
},
|
||
|
|
"n-shot": {
|
||
|
|
"mmlu": 0,
|
||
|
|
"mmlu_abstract_algebra": 5,
|
||
|
|
"mmlu_anatomy": 5,
|
||
|
|
"mmlu_astronomy": 5,
|
||
|
|
"mmlu_business_ethics": 5,
|
||
|
|
"mmlu_clinical_knowledge": 5,
|
||
|
|
"mmlu_college_biology": 5,
|
||
|
|
"mmlu_college_chemistry": 5,
|
||
|
|
"mmlu_college_computer_science": 5,
|
||
|
|
"mmlu_college_mathematics": 5,
|
||
|
|
"mmlu_college_medicine": 5,
|
||
|
|
"mmlu_college_physics": 5,
|
||
|
|
"mmlu_computer_security": 5,
|
||
|
|
"mmlu_conceptual_physics": 5,
|
||
|
|
"mmlu_econometrics": 5,
|
||
|
|
"mmlu_electrical_engineering": 5,
|
||
|
|
"mmlu_elementary_mathematics": 5,
|
||
|
|
"mmlu_formal_logic": 5,
|
||
|
|
"mmlu_global_facts": 5,
|
||
|
|
"mmlu_high_school_biology": 5,
|
||
|
|
"mmlu_high_school_chemistry": 5,
|
||
|
|
"mmlu_high_school_computer_science": 5,
|
||
|
|
"mmlu_high_school_european_history": 5,
|
||
|
|
"mmlu_high_school_geography": 5,
|
||
|
|
"mmlu_high_school_government_and_politics": 5,
|
||
|
|
"mmlu_high_school_macroeconomics": 5,
|
||
|
|
"mmlu_high_school_mathematics": 5,
|
||
|
|
"mmlu_high_school_microeconomics": 5,
|
||
|
|
"mmlu_high_school_physics": 5,
|
||
|
|
"mmlu_high_school_psychology": 5,
|
||
|
|
"mmlu_high_school_statistics": 5,
|
||
|
|
"mmlu_high_school_us_history": 5,
|
||
|
|
"mmlu_high_school_world_history": 5,
|
||
|
|
"mmlu_human_aging": 5,
|
||
|
|
"mmlu_human_sexuality": 5,
|
||
|
|
"mmlu_humanities": 5,
|
||
|
|
"mmlu_international_law": 5,
|
||
|
|
"mmlu_jurisprudence": 5,
|
||
|
|
"mmlu_logical_fallacies": 5,
|
||
|
|
"mmlu_machine_learning": 5,
|
||
|
|
"mmlu_management": 5,
|
||
|
|
"mmlu_marketing": 5,
|
||
|
|
"mmlu_medical_genetics": 5,
|
||
|
|
"mmlu_miscellaneous": 5,
|
||
|
|
"mmlu_moral_disputes": 5,
|
||
|
|
"mmlu_moral_scenarios": 5,
|
||
|
|
"mmlu_nutrition": 5,
|
||
|
|
"mmlu_other": 5,
|
||
|
|
"mmlu_philosophy": 5,
|
||
|
|
"mmlu_prehistory": 5,
|
||
|
|
"mmlu_professional_accounting": 5,
|
||
|
|
"mmlu_professional_law": 5,
|
||
|
|
"mmlu_professional_medicine": 5,
|
||
|
|
"mmlu_professional_psychology": 5,
|
||
|
|
"mmlu_public_relations": 5,
|
||
|
|
"mmlu_security_studies": 5,
|
||
|
|
"mmlu_social_sciences": 5,
|
||
|
|
"mmlu_sociology": 5,
|
||
|
|
"mmlu_stem": 5,
|
||
|
|
"mmlu_us_foreign_policy": 5,
|
||
|
|
"mmlu_virology": 5,
|
||
|
|
"mmlu_world_religions": 5
|
||
|
|
},
|
||
|
|
"config": {
|
||
|
|
"model": "hf",
|
||
|
|
"model_args": "pretrained=/home/zeus/microllm2/microllm2-checkpoints/final_merged,dtype=bfloat16,trust_remote_code=False",
|
||
|
|
"batch_size": "8",
|
||
|
|
"batch_sizes": [],
|
||
|
|
"device": "cuda:0",
|
||
|
|
"use_cache": null,
|
||
|
|
"limit": null,
|
||
|
|
"bootstrap_iters": 100000,
|
||
|
|
"gen_kwargs": null
|
||
|
|
},
|
||
|
|
"git_hash": null,
|
||
|
|
"date": 1786317646.6679428,
|
||
|
|
"pretty_env_info": "PyTorch version: 2.4.1+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 24.04.3 LTS (x86_64)\nGCC version: (Ubuntu 13.3.0-6ubuntu2~24.04) 13.3.0\nClang version: Could not collect\nCMake version: version 3.28.3\nLibc version: glibc-2.39\n\nPython version: 3.12.11 | packaged by Anaconda, Inc. | (main, Jun 5 2025, 13:09:17) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-6.8.0-110-generic-x86_64-with-glibc2.39\nIs CUDA available: True\nCUDA runtime version: 13.0.88\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: GPU 0: NVIDIA H100 80GB HBM3\nNvidia driver version: 580.173.02\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.14.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.14.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.14.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.14.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.14.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.14.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.14.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.14.0\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 24\nOn-line CPU(s) list: 0-23\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8470\nCPU family: 6\nModel: 143\nThread(s) per core: 2\nCore(s) per socket: 12\nSocket(s): 1\nStepping: 8\nBogoMIPS: 4000.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid tsc_known_freq pni pclmulqdq dtes64 ssse3 fma cx16 pdcm pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch cpuid_fault ssbd ibrs ibpb stibp ibrs_enhanced fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves avx_vnni avx512_bf16 wbnoinvd arat avx512vbmi umip pku ospke waitpkg avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq la57 rdpid bus_lock_detect cldemote movdiri movdir64b fsrm md_clear serialize tsxldtrk amx_bf16 avx512_fp16 amx_tile amx_int8 flush_l1d arch_capabilities\nHypervisor vendor: KVM\nVirtualization type: full\nL1d cache: 576 KiB (12 instances)\nL1i cache: 384 KiB (12 instances)\nL2 cache: 24 MiB (12 instances)\nL3 cache: 105 MiB (1 instance)\nNUMA node(s): 1\nNUMA node0 CPU(s): 0-23\nVulnerability Gather data sampling: Not affected\nVulnerability Indirect target selection: Not affected\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Not affected\nVulnerability Reg file data sampling: Not affected\nVulnerability Retbleed: Not affected\nVulnerability Spec rstack overflow: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl\nVulnera
|
||
|
|
"transformers_version": "4.44.2",
|
||
|
|
"upper_git_hash": null
|
||
|
|
}
|