3683 lines
108 KiB
JSON
3683 lines
108 KiB
JSON
{
|
|
"results": {
|
|
"arc_challenge": {
|
|
"alias": "arc_challenge",
|
|
"acc,none": 0.28754266211604096,
|
|
"acc_stderr,none": 0.013226719056266125,
|
|
"acc_norm,none": 0.3174061433447099,
|
|
"acc_norm_stderr,none": 0.013602239088038169
|
|
},
|
|
"boolq": {
|
|
"alias": "boolq",
|
|
"acc,none": 0.7437308868501529,
|
|
"acc_stderr,none": 0.007635695159015045
|
|
},
|
|
"gsm8k": {
|
|
"alias": "gsm8k",
|
|
"exact_match,strict-match": 0.620166793025019,
|
|
"exact_match_stderr,strict-match": 0.013368818096960505,
|
|
"exact_match,flexible-extract": 0.6239575435936315,
|
|
"exact_match_stderr,flexible-extract": 0.013342532064849763
|
|
},
|
|
"hellaswag": {
|
|
"alias": "hellaswag",
|
|
"acc,none": 0.4291973710416252,
|
|
"acc_stderr,none": 0.0049395004048821845,
|
|
"acc_norm,none": 0.560744871539534,
|
|
"acc_norm_stderr,none": 0.004952820538831937
|
|
},
|
|
"mmlu": {
|
|
"acc,none": 0.5997721122347244,
|
|
"acc_stderr,none": 0.003956800884450581,
|
|
"alias": "mmlu"
|
|
},
|
|
"mmlu_humanities": {
|
|
"acc,none": 0.5300743889479277,
|
|
"acc_stderr,none": 0.006827554102250098,
|
|
"alias": " - humanities"
|
|
},
|
|
"mmlu_formal_logic": {
|
|
"alias": " - formal_logic",
|
|
"acc,none": 0.5,
|
|
"acc_stderr,none": 0.04472135954999579
|
|
},
|
|
"mmlu_high_school_european_history": {
|
|
"alias": " - high_school_european_history",
|
|
"acc,none": 0.7696969696969697,
|
|
"acc_stderr,none": 0.03287666758603488
|
|
},
|
|
"mmlu_high_school_us_history": {
|
|
"alias": " - high_school_us_history",
|
|
"acc,none": 0.7156862745098039,
|
|
"acc_stderr,none": 0.03166009679399812
|
|
},
|
|
"mmlu_high_school_world_history": {
|
|
"alias": " - high_school_world_history",
|
|
"acc,none": 0.7974683544303798,
|
|
"acc_stderr,none": 0.026160568246601464
|
|
},
|
|
"mmlu_international_law": {
|
|
"alias": " - international_law",
|
|
"acc,none": 0.7851239669421488,
|
|
"acc_stderr,none": 0.03749492448709699
|
|
},
|
|
"mmlu_jurisprudence": {
|
|
"alias": " - jurisprudence",
|
|
"acc,none": 0.7222222222222222,
|
|
"acc_stderr,none": 0.043300437496507416
|
|
},
|
|
"mmlu_logical_fallacies": {
|
|
"alias": " - logical_fallacies",
|
|
"acc,none": 0.6932515337423313,
|
|
"acc_stderr,none": 0.036230899157241474
|
|
},
|
|
"mmlu_moral_disputes": {
|
|
"alias": " - moral_disputes",
|
|
"acc,none": 0.5953757225433526,
|
|
"acc_stderr,none": 0.026424816594009852
|
|
},
|
|
"mmlu_moral_scenarios": {
|
|
"alias": " - moral_scenarios",
|
|
"acc,none": 0.2446927374301676,
|
|
"acc_stderr,none": 0.014378169884098417
|
|
},
|
|
"mmlu_philosophy": {
|
|
"alias": " - philosophy",
|
|
"acc,none": 0.6559485530546624,
|
|
"acc_stderr,none": 0.026981478043648043
|
|
},
|
|
"mmlu_prehistory": {
|
|
"alias": " - prehistory",
|
|
"acc,none": 0.6265432098765432,
|
|
"acc_stderr,none": 0.026915003011380154
|
|
},
|
|
"mmlu_professional_law": {
|
|
"alias": " - professional_law",
|
|
"acc,none": 0.4745762711864407,
|
|
"acc_stderr,none": 0.012753716929100998
|
|
},
|
|
"mmlu_world_religions": {
|
|
"alias": " - world_religions",
|
|
"acc,none": 0.7192982456140351,
|
|
"acc_stderr,none": 0.034462962170884265
|
|
},
|
|
"mmlu_other": {
|
|
"acc,none": 0.6269713550048278,
|
|
"acc_stderr,none": 0.008479456967258268,
|
|
"alias": " - other"
|
|
},
|
|
"mmlu_business_ethics": {
|
|
"alias": " - business_ethics",
|
|
"acc,none": 0.62,
|
|
"acc_stderr,none": 0.048783173121456316
|
|
},
|
|
"mmlu_clinical_knowledge": {
|
|
"alias": " - clinical_knowledge",
|
|
"acc,none": 0.6415094339622641,
|
|
"acc_stderr,none": 0.02951470358398177
|
|
},
|
|
"mmlu_college_medicine": {
|
|
"alias": " - college_medicine",
|
|
"acc,none": 0.5953757225433526,
|
|
"acc_stderr,none": 0.03742461193887248
|
|
},
|
|
"mmlu_global_facts": {
|
|
"alias": " - global_facts",
|
|
"acc,none": 0.33,
|
|
"acc_stderr,none": 0.04725815626252605
|
|
},
|
|
"mmlu_human_aging": {
|
|
"alias": " - human_aging",
|
|
"acc,none": 0.600896860986547,
|
|
"acc_stderr,none": 0.032867453125679603
|
|
},
|
|
"mmlu_management": {
|
|
"alias": " - management",
|
|
"acc,none": 0.6893203883495146,
|
|
"acc_stderr,none": 0.04582124160161549
|
|
},
|
|
"mmlu_marketing": {
|
|
"alias": " - marketing",
|
|
"acc,none": 0.8034188034188035,
|
|
"acc_stderr,none": 0.02603538609895129
|
|
},
|
|
"mmlu_medical_genetics": {
|
|
"alias": " - medical_genetics",
|
|
"acc,none": 0.69,
|
|
"acc_stderr,none": 0.04648231987117316
|
|
},
|
|
"mmlu_miscellaneous": {
|
|
"alias": " - miscellaneous",
|
|
"acc,none": 0.6717752234993615,
|
|
"acc_stderr,none": 0.016791685640192892
|
|
},
|
|
"mmlu_nutrition": {
|
|
"alias": " - nutrition",
|
|
"acc,none": 0.6764705882352942,
|
|
"acc_stderr,none": 0.0267874531119065
|
|
},
|
|
"mmlu_professional_accounting": {
|
|
"alias": " - professional_accounting",
|
|
"acc,none": 0.4397163120567376,
|
|
"acc_stderr,none": 0.02960991207559411
|
|
},
|
|
"mmlu_professional_medicine": {
|
|
"alias": " - professional_medicine",
|
|
"acc,none": 0.6838235294117647,
|
|
"acc_stderr,none": 0.02824568739146291
|
|
},
|
|
"mmlu_virology": {
|
|
"alias": " - virology",
|
|
"acc,none": 0.45180722891566266,
|
|
"acc_stderr,none": 0.03874371556587953
|
|
},
|
|
"mmlu_social_sciences": {
|
|
"acc,none": 0.6906077348066298,
|
|
"acc_stderr,none": 0.008146459902876074,
|
|
"alias": " - social sciences"
|
|
},
|
|
"mmlu_econometrics": {
|
|
"alias": " - econometrics",
|
|
"acc,none": 0.35964912280701755,
|
|
"acc_stderr,none": 0.04514496132873633
|
|
},
|
|
"mmlu_high_school_geography": {
|
|
"alias": " - high_school_geography",
|
|
"acc,none": 0.7272727272727273,
|
|
"acc_stderr,none": 0.03173071239071724
|
|
},
|
|
"mmlu_high_school_government_and_politics": {
|
|
"alias": " - high_school_government_and_politics",
|
|
"acc,none": 0.7461139896373057,
|
|
"acc_stderr,none": 0.0314102478056532
|
|
},
|
|
"mmlu_high_school_macroeconomics": {
|
|
"alias": " - high_school_macroeconomics",
|
|
"acc,none": 0.6435897435897436,
|
|
"acc_stderr,none": 0.0242831405294673
|
|
},
|
|
"mmlu_high_school_microeconomics": {
|
|
"alias": " - high_school_microeconomics",
|
|
"acc,none": 0.7773109243697479,
|
|
"acc_stderr,none": 0.027025433498882374
|
|
},
|
|
"mmlu_high_school_psychology": {
|
|
"alias": " - high_school_psychology",
|
|
"acc,none": 0.8,
|
|
"acc_stderr,none": 0.017149858514250927
|
|
},
|
|
"mmlu_human_sexuality": {
|
|
"alias": " - human_sexuality",
|
|
"acc,none": 0.6946564885496184,
|
|
"acc_stderr,none": 0.040393149787245626
|
|
},
|
|
"mmlu_professional_psychology": {
|
|
"alias": " - professional_psychology",
|
|
"acc,none": 0.5915032679738562,
|
|
"acc_stderr,none": 0.01988622103750188
|
|
},
|
|
"mmlu_public_relations": {
|
|
"alias": " - public_relations",
|
|
"acc,none": 0.6,
|
|
"acc_stderr,none": 0.0469237132203465
|
|
},
|
|
"mmlu_security_studies": {
|
|
"alias": " - security_studies",
|
|
"acc,none": 0.7020408163265306,
|
|
"acc_stderr,none": 0.02927956741106567
|
|
},
|
|
"mmlu_sociology": {
|
|
"alias": " - sociology",
|
|
"acc,none": 0.7711442786069652,
|
|
"acc_stderr,none": 0.029705284056772443
|
|
},
|
|
"mmlu_us_foreign_policy": {
|
|
"alias": " - us_foreign_policy",
|
|
"acc,none": 0.78,
|
|
"acc_stderr,none": 0.04163331998932261
|
|
},
|
|
"mmlu_stem": {
|
|
"acc,none": 0.5883285759594037,
|
|
"acc_stderr,none": 0.008585558598510803,
|
|
"alias": " - stem"
|
|
},
|
|
"mmlu_abstract_algebra": {
|
|
"alias": " - abstract_algebra",
|
|
"acc,none": 0.43,
|
|
"acc_stderr,none": 0.04975698519562428
|
|
},
|
|
"mmlu_anatomy": {
|
|
"alias": " - anatomy",
|
|
"acc,none": 0.6074074074074074,
|
|
"acc_stderr,none": 0.04218506215368879
|
|
},
|
|
"mmlu_astronomy": {
|
|
"alias": " - astronomy",
|
|
"acc,none": 0.6973684210526315,
|
|
"acc_stderr,none": 0.03738520676119667
|
|
},
|
|
"mmlu_college_biology": {
|
|
"alias": " - college_biology",
|
|
"acc,none": 0.8263888888888888,
|
|
"acc_stderr,none": 0.03167473383795717
|
|
},
|
|
"mmlu_college_chemistry": {
|
|
"alias": " - college_chemistry",
|
|
"acc,none": 0.53,
|
|
"acc_stderr,none": 0.050161355804659205
|
|
},
|
|
"mmlu_college_computer_science": {
|
|
"alias": " - college_computer_science",
|
|
"acc,none": 0.54,
|
|
"acc_stderr,none": 0.05009082659620332
|
|
},
|
|
"mmlu_college_mathematics": {
|
|
"alias": " - college_mathematics",
|
|
"acc,none": 0.5,
|
|
"acc_stderr,none": 0.050251890762960605
|
|
},
|
|
"mmlu_college_physics": {
|
|
"alias": " - college_physics",
|
|
"acc,none": 0.5,
|
|
"acc_stderr,none": 0.04975185951049946
|
|
},
|
|
"mmlu_computer_security": {
|
|
"alias": " - computer_security",
|
|
"acc,none": 0.68,
|
|
"acc_stderr,none": 0.04688261722621504
|
|
},
|
|
"mmlu_conceptual_physics": {
|
|
"alias": " - conceptual_physics",
|
|
"acc,none": 0.5872340425531914,
|
|
"acc_stderr,none": 0.03218471141400351
|
|
},
|
|
"mmlu_electrical_engineering": {
|
|
"alias": " - electrical_engineering",
|
|
"acc,none": 0.6413793103448275,
|
|
"acc_stderr,none": 0.03996629574876719
|
|
},
|
|
"mmlu_elementary_mathematics": {
|
|
"alias": " - elementary_mathematics",
|
|
"acc,none": 0.5317460317460317,
|
|
"acc_stderr,none": 0.025699352832131792
|
|
},
|
|
"mmlu_high_school_biology": {
|
|
"alias": " - high_school_biology",
|
|
"acc,none": 0.7548387096774194,
|
|
"acc_stderr,none": 0.02447224384089551
|
|
},
|
|
"mmlu_high_school_chemistry": {
|
|
"alias": " - high_school_chemistry",
|
|
"acc,none": 0.6009852216748769,
|
|
"acc_stderr,none": 0.034454876862647144
|
|
},
|
|
"mmlu_high_school_computer_science": {
|
|
"alias": " - high_school_computer_science",
|
|
"acc,none": 0.69,
|
|
"acc_stderr,none": 0.04648231987117316
|
|
},
|
|
"mmlu_high_school_mathematics": {
|
|
"alias": " - high_school_mathematics",
|
|
"acc,none": 0.45555555555555555,
|
|
"acc_stderr,none": 0.03036486250482443
|
|
},
|
|
"mmlu_high_school_physics": {
|
|
"alias": " - high_school_physics",
|
|
"acc,none": 0.5165562913907285,
|
|
"acc_stderr,none": 0.04080244185628972
|
|
},
|
|
"mmlu_high_school_statistics": {
|
|
"alias": " - high_school_statistics",
|
|
"acc,none": 0.5694444444444444,
|
|
"acc_stderr,none": 0.03376922151252336
|
|
},
|
|
"mmlu_machine_learning": {
|
|
"alias": " - machine_learning",
|
|
"acc,none": 0.42857142857142855,
|
|
"acc_stderr,none": 0.04697113923010212
|
|
},
|
|
"piqa": {
|
|
"alias": "piqa",
|
|
"acc,none": 0.6327529923830251,
|
|
"acc_stderr,none": 0.011247128539690562,
|
|
"acc_norm,none": 0.6322089227421109,
|
|
"acc_norm_stderr,none": 0.011250616646678792
|
|
},
|
|
"truthfulqa_mc2": {
|
|
"alias": "truthfulqa_mc2",
|
|
"acc,none": 0.45340473177307805,
|
|
"acc_stderr,none": 0.01602761584945673
|
|
},
|
|
"winogrande": {
|
|
"alias": "winogrande",
|
|
"acc,none": 0.500394632991318,
|
|
"acc_stderr,none": 0.014052481306049516
|
|
}
|
|
},
|
|
"groups": {
|
|
"mmlu": {
|
|
"acc,none": 0.5997721122347244,
|
|
"acc_stderr,none": 0.003956800884450581,
|
|
"alias": "mmlu"
|
|
},
|
|
"mmlu_humanities": {
|
|
"acc,none": 0.5300743889479277,
|
|
"acc_stderr,none": 0.006827554102250098,
|
|
"alias": " - humanities"
|
|
},
|
|
"mmlu_other": {
|
|
"acc,none": 0.6269713550048278,
|
|
"acc_stderr,none": 0.008479456967258268,
|
|
"alias": " - other"
|
|
},
|
|
"mmlu_social_sciences": {
|
|
"acc,none": 0.6906077348066298,
|
|
"acc_stderr,none": 0.008146459902876074,
|
|
"alias": " - social sciences"
|
|
},
|
|
"mmlu_stem": {
|
|
"acc,none": 0.5883285759594037,
|
|
"acc_stderr,none": 0.008585558598510803,
|
|
"alias": " - stem"
|
|
}
|
|
},
|
|
"group_subtasks": {
|
|
"arc_challenge": [],
|
|
"boolq": [],
|
|
"gsm8k": [],
|
|
"hellaswag": [],
|
|
"mmlu_humanities": [
|
|
"mmlu_formal_logic",
|
|
"mmlu_high_school_european_history",
|
|
"mmlu_high_school_us_history",
|
|
"mmlu_high_school_world_history",
|
|
"mmlu_international_law",
|
|
"mmlu_jurisprudence",
|
|
"mmlu_logical_fallacies",
|
|
"mmlu_moral_disputes",
|
|
"mmlu_moral_scenarios",
|
|
"mmlu_philosophy",
|
|
"mmlu_prehistory",
|
|
"mmlu_professional_law",
|
|
"mmlu_world_religions"
|
|
],
|
|
"mmlu_social_sciences": [
|
|
"mmlu_econometrics",
|
|
"mmlu_high_school_geography",
|
|
"mmlu_high_school_government_and_politics",
|
|
"mmlu_high_school_macroeconomics",
|
|
"mmlu_high_school_microeconomics",
|
|
"mmlu_high_school_psychology",
|
|
"mmlu_human_sexuality",
|
|
"mmlu_professional_psychology",
|
|
"mmlu_public_relations",
|
|
"mmlu_security_studies",
|
|
"mmlu_sociology",
|
|
"mmlu_us_foreign_policy"
|
|
],
|
|
"mmlu_other": [
|
|
"mmlu_business_ethics",
|
|
"mmlu_clinical_knowledge",
|
|
"mmlu_college_medicine",
|
|
"mmlu_global_facts",
|
|
"mmlu_human_aging",
|
|
"mmlu_management",
|
|
"mmlu_marketing",
|
|
"mmlu_medical_genetics",
|
|
"mmlu_miscellaneous",
|
|
"mmlu_nutrition",
|
|
"mmlu_professional_accounting",
|
|
"mmlu_professional_medicine",
|
|
"mmlu_virology"
|
|
],
|
|
"mmlu_stem": [
|
|
"mmlu_abstract_algebra",
|
|
"mmlu_anatomy",
|
|
"mmlu_astronomy",
|
|
"mmlu_college_biology",
|
|
"mmlu_college_chemistry",
|
|
"mmlu_college_computer_science",
|
|
"mmlu_college_mathematics",
|
|
"mmlu_college_physics",
|
|
"mmlu_computer_security",
|
|
"mmlu_conceptual_physics",
|
|
"mmlu_electrical_engineering",
|
|
"mmlu_elementary_mathematics",
|
|
"mmlu_high_school_biology",
|
|
"mmlu_high_school_chemistry",
|
|
"mmlu_high_school_computer_science",
|
|
"mmlu_high_school_mathematics",
|
|
"mmlu_high_school_physics",
|
|
"mmlu_high_school_statistics",
|
|
"mmlu_machine_learning"
|
|
],
|
|
"mmlu": [
|
|
"mmlu_stem",
|
|
"mmlu_other",
|
|
"mmlu_social_sciences",
|
|
"mmlu_humanities"
|
|
],
|
|
"piqa": [],
|
|
"truthfulqa_mc2": [],
|
|
"winogrande": []
|
|
},
|
|
"configs": {
|
|
"arc_challenge": {
|
|
"task": "arc_challenge",
|
|
"tag": [
|
|
"ai2_arc"
|
|
],
|
|
"dataset_path": "allenai/ai2_arc",
|
|
"dataset_name": "ARC-Challenge",
|
|
"training_split": "train",
|
|
"validation_split": "validation",
|
|
"test_split": "test",
|
|
"doc_to_text": "Question: {{question}}\nAnswer:",
|
|
"doc_to_target": "{{choices.label.index(answerKey)}}",
|
|
"doc_to_choice": "{{choices.text}}",
|
|
"description": "",
|
|
"target_delimiter": " ",
|
|
"fewshot_delimiter": "\n\n",
|
|
"num_fewshot": 0,
|
|
"metric_list": [
|
|
{
|
|
"metric": "acc",
|
|
"aggregation": "mean",
|
|
"higher_is_better": true
|
|
},
|
|
{
|
|
"metric": "acc_norm",
|
|
"aggregation": "mean",
|
|
"higher_is_better": true
|
|
}
|
|
],
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": true,
|
|
"doc_to_decontamination_query": "Question: {{question}}\nAnswer:",
|
|
"metadata": {
|
|
"version": 1.0
|
|
}
|
|
},
|
|
"boolq": {
|
|
"task": "boolq",
|
|
"tag": [
|
|
"super-glue-lm-eval-v1"
|
|
],
|
|
"dataset_path": "super_glue",
|
|
"dataset_name": "boolq",
|
|
"training_split": "train",
|
|
"validation_split": "validation",
|
|
"doc_to_text": "{{passage}}\nQuestion: {{question}}?\nAnswer:",
|
|
"doc_to_target": "label",
|
|
"doc_to_choice": [
|
|
"no",
|
|
"yes"
|
|
],
|
|
"description": "",
|
|
"target_delimiter": " ",
|
|
"fewshot_delimiter": "\n\n",
|
|
"num_fewshot": 0,
|
|
"metric_list": [
|
|
{
|
|
"metric": "acc"
|
|
}
|
|
],
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": true,
|
|
"doc_to_decontamination_query": "passage",
|
|
"metadata": {
|
|
"version": 2.0
|
|
}
|
|
},
|
|
"gsm8k": {
|
|
"task": "gsm8k",
|
|
"tag": [
|
|
"math_word_problems"
|
|
],
|
|
"dataset_path": "gsm8k",
|
|
"dataset_name": "main",
|
|
"training_split": "train",
|
|
"test_split": "test",
|
|
"fewshot_split": "train",
|
|
"doc_to_text": "Question: {{question}}\nAnswer:",
|
|
"doc_to_target": "{{answer}}",
|
|
"description": "",
|
|
"target_delimiter": " ",
|
|
"fewshot_delimiter": "\n\n",
|
|
"num_fewshot": 5,
|
|
"metric_list": [
|
|
{
|
|
"metric": "exact_match",
|
|
"aggregation": "mean",
|
|
"higher_is_better": true,
|
|
"ignore_case": true,
|
|
"ignore_punctuation": false,
|
|
"regexes_to_ignore": [
|
|
",",
|
|
"\\$",
|
|
"(?s).*#### ",
|
|
"\\.$"
|
|
]
|
|
}
|
|
],
|
|
"output_type": "generate_until",
|
|
"generation_kwargs": {
|
|
"until": [
|
|
"Question:",
|
|
"</s>",
|
|
"<|im_end|>"
|
|
],
|
|
"do_sample": false,
|
|
"temperature": 0.0
|
|
},
|
|
"repeats": 1,
|
|
"filter_list": [
|
|
{
|
|
"name": "strict-match",
|
|
"filter": [
|
|
{
|
|
"function": "regex",
|
|
"regex_pattern": "#### (\\-?[0-9\\.\\,]+)"
|
|
},
|
|
{
|
|
"function": "take_first"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"name": "flexible-extract",
|
|
"filter": [
|
|
{
|
|
"function": "regex",
|
|
"group_select": -1,
|
|
"regex_pattern": "(-?[$0-9.,]{2,})|(-?[0-9]+)"
|
|
},
|
|
{
|
|
"function": "take_first"
|
|
}
|
|
]
|
|
}
|
|
],
|
|
"should_decontaminate": false,
|
|
"metadata": {
|
|
"version": 3.0
|
|
}
|
|
},
|
|
"hellaswag": {
|
|
"task": "hellaswag",
|
|
"tag": [
|
|
"multiple_choice"
|
|
],
|
|
"dataset_path": "hellaswag",
|
|
"dataset_kwargs": {
|
|
"trust_remote_code": true
|
|
},
|
|
"training_split": "train",
|
|
"validation_split": "validation",
|
|
"process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n out_doc = {\n \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n \"gold\": int(doc[\"label\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n",
|
|
"doc_to_text": "{{query}}",
|
|
"doc_to_target": "{{label}}",
|
|
"doc_to_choice": "choices",
|
|
"description": "",
|
|
"target_delimiter": " ",
|
|
"fewshot_delimiter": "\n\n",
|
|
"num_fewshot": 0,
|
|
"metric_list": [
|
|
{
|
|
"metric": "acc",
|
|
"aggregation": "mean",
|
|
"higher_is_better": true
|
|
},
|
|
{
|
|
"metric": "acc_norm",
|
|
"aggregation": "mean",
|
|
"higher_is_better": true
|
|
}
|
|
],
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"metadata": {
|
|
"version": 1.0
|
|
}
|
|
},
|
|
"mmlu_abstract_algebra": {
|
|
"task": "mmlu_abstract_algebra",
|
|
"task_alias": "abstract_algebra",
|
|
"tag": "mmlu_stem_tasks",
|
|
"dataset_path": "hails/mmlu_no_train",
|
|
"dataset_name": "abstract_algebra",
|
|
"dataset_kwargs": {
|
|
"trust_remote_code": true
|
|
},
|
|
"test_split": "test",
|
|
"fewshot_split": "dev",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"doc_to_target": "answer",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"description": "The following are multiple choice questions (with answers) about abstract algebra.\n\n",
|
|
"target_delimiter": " ",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_config": {
|
|
"sampler": "first_n"
|
|
},
|
|
"num_fewshot": 0,
|
|
"metric_list": [
|
|
{
|
|
"metric": "acc",
|
|
"aggregation": "mean",
|
|
"higher_is_better": true
|
|
}
|
|
],
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"metadata": {
|
|
"version": 1.0
|
|
}
|
|
},
|
|
"mmlu_anatomy": {
|
|
"task": "mmlu_anatomy",
|
|
"task_alias": "anatomy",
|
|
"tag": "mmlu_stem_tasks",
|
|
"dataset_path": "hails/mmlu_no_train",
|
|
"dataset_name": "anatomy",
|
|
"dataset_kwargs": {
|
|
"trust_remote_code": true
|
|
},
|
|
"test_split": "test",
|
|
"fewshot_split": "dev",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"doc_to_target": "answer",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"description": "The following are multiple choice questions (with answers) about anatomy.\n\n",
|
|
"target_delimiter": " ",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_config": {
|
|
"sampler": "first_n"
|
|
},
|
|
"num_fewshot": 0,
|
|
"metric_list": [
|
|
{
|
|
"metric": "acc",
|
|
"aggregation": "mean",
|
|
"higher_is_better": true
|
|
}
|
|
],
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"metadata": {
|
|
"version": 1.0
|
|
}
|
|
},
|
|
"mmlu_astronomy": {
|
|
"task": "mmlu_astronomy",
|
|
"task_alias": "astronomy",
|
|
"tag": "mmlu_stem_tasks",
|
|
"dataset_path": "hails/mmlu_no_train",
|
|
"dataset_name": "astronomy",
|
|
"dataset_kwargs": {
|
|
"trust_remote_code": true
|
|
},
|
|
"test_split": "test",
|
|
"fewshot_split": "dev",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"doc_to_target": "answer",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"description": "The following are multiple choice questions (with answers) about astronomy.\n\n",
|
|
"target_delimiter": " ",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_config": {
|
|
"sampler": "first_n"
|
|
},
|
|
"num_fewshot": 0,
|
|
"metric_list": [
|
|
{
|
|
"metric": "acc",
|
|
"aggregation": "mean",
|
|
"higher_is_better": true
|
|
}
|
|
],
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"metadata": {
|
|
"version": 1.0
|
|
}
|
|
},
|
|
"mmlu_business_ethics": {
|
|
"task": "mmlu_business_ethics",
|
|
"task_alias": "business_ethics",
|
|
"tag": "mmlu_other_tasks",
|
|
"dataset_path": "hails/mmlu_no_train",
|
|
"dataset_name": "business_ethics",
|
|
"dataset_kwargs": {
|
|
"trust_remote_code": true
|
|
},
|
|
"test_split": "test",
|
|
"fewshot_split": "dev",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"doc_to_target": "answer",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"description": "The following are multiple choice questions (with answers) about business ethics.\n\n",
|
|
"target_delimiter": " ",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_config": {
|
|
"sampler": "first_n"
|
|
},
|
|
"num_fewshot": 0,
|
|
"metric_list": [
|
|
{
|
|
"metric": "acc",
|
|
"aggregation": "mean",
|
|
"higher_is_better": true
|
|
}
|
|
],
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"metadata": {
|
|
"version": 1.0
|
|
}
|
|
},
|
|
"mmlu_clinical_knowledge": {
|
|
"task": "mmlu_clinical_knowledge",
|
|
"task_alias": "clinical_knowledge",
|
|
"tag": "mmlu_other_tasks",
|
|
"dataset_path": "hails/mmlu_no_train",
|
|
"dataset_name": "clinical_knowledge",
|
|
"dataset_kwargs": {
|
|
"trust_remote_code": true
|
|
},
|
|
"test_split": "test",
|
|
"fewshot_split": "dev",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"doc_to_target": "answer",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"description": "The following are multiple choice questions (with answers) about clinical knowledge.\n\n",
|
|
"target_delimiter": " ",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_config": {
|
|
"sampler": "first_n"
|
|
},
|
|
"num_fewshot": 0,
|
|
"metric_list": [
|
|
{
|
|
"metric": "acc",
|
|
"aggregation": "mean",
|
|
"higher_is_better": true
|
|
}
|
|
],
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"metadata": {
|
|
"version": 1.0
|
|
}
|
|
},
|
|
"mmlu_college_biology": {
|
|
"task": "mmlu_college_biology",
|
|
"task_alias": "college_biology",
|
|
"tag": "mmlu_stem_tasks",
|
|
"dataset_path": "hails/mmlu_no_train",
|
|
"dataset_name": "college_biology",
|
|
"dataset_kwargs": {
|
|
"trust_remote_code": true
|
|
},
|
|
"test_split": "test",
|
|
"fewshot_split": "dev",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"doc_to_target": "answer",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"description": "The following are multiple choice questions (with answers) about college biology.\n\n",
|
|
"target_delimiter": " ",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_config": {
|
|
"sampler": "first_n"
|
|
},
|
|
"num_fewshot": 0,
|
|
"metric_list": [
|
|
{
|
|
"metric": "acc",
|
|
"aggregation": "mean",
|
|
"higher_is_better": true
|
|
}
|
|
],
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"metadata": {
|
|
"version": 1.0
|
|
}
|
|
},
|
|
"mmlu_college_chemistry": {
|
|
"task": "mmlu_college_chemistry",
|
|
"task_alias": "college_chemistry",
|
|
"tag": "mmlu_stem_tasks",
|
|
"dataset_path": "hails/mmlu_no_train",
|
|
"dataset_name": "college_chemistry",
|
|
"dataset_kwargs": {
|
|
"trust_remote_code": true
|
|
},
|
|
"test_split": "test",
|
|
"fewshot_split": "dev",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"doc_to_target": "answer",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"description": "The following are multiple choice questions (with answers) about college chemistry.\n\n",
|
|
"target_delimiter": " ",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_config": {
|
|
"sampler": "first_n"
|
|
},
|
|
"num_fewshot": 0,
|
|
"metric_list": [
|
|
{
|
|
"metric": "acc",
|
|
"aggregation": "mean",
|
|
"higher_is_better": true
|
|
}
|
|
],
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"metadata": {
|
|
"version": 1.0
|
|
}
|
|
},
|
|
"mmlu_college_computer_science": {
|
|
"task": "mmlu_college_computer_science",
|
|
"task_alias": "college_computer_science",
|
|
"tag": "mmlu_stem_tasks",
|
|
"dataset_path": "hails/mmlu_no_train",
|
|
"dataset_name": "college_computer_science",
|
|
"dataset_kwargs": {
|
|
"trust_remote_code": true
|
|
},
|
|
"test_split": "test",
|
|
"fewshot_split": "dev",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"doc_to_target": "answer",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"description": "The following are multiple choice questions (with answers) about college computer science.\n\n",
|
|
"target_delimiter": " ",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_config": {
|
|
"sampler": "first_n"
|
|
},
|
|
"num_fewshot": 0,
|
|
"metric_list": [
|
|
{
|
|
"metric": "acc",
|
|
"aggregation": "mean",
|
|
"higher_is_better": true
|
|
}
|
|
],
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"metadata": {
|
|
"version": 1.0
|
|
}
|
|
},
|
|
"mmlu_college_mathematics": {
|
|
"task": "mmlu_college_mathematics",
|
|
"task_alias": "college_mathematics",
|
|
"tag": "mmlu_stem_tasks",
|
|
"dataset_path": "hails/mmlu_no_train",
|
|
"dataset_name": "college_mathematics",
|
|
"dataset_kwargs": {
|
|
"trust_remote_code": true
|
|
},
|
|
"test_split": "test",
|
|
"fewshot_split": "dev",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"doc_to_target": "answer",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"description": "The following are multiple choice questions (with answers) about college mathematics.\n\n",
|
|
"target_delimiter": " ",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_config": {
|
|
"sampler": "first_n"
|
|
},
|
|
"num_fewshot": 0,
|
|
"metric_list": [
|
|
{
|
|
"metric": "acc",
|
|
"aggregation": "mean",
|
|
"higher_is_better": true
|
|
}
|
|
],
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"metadata": {
|
|
"version": 1.0
|
|
}
|
|
},
|
|
"mmlu_college_medicine": {
|
|
"task": "mmlu_college_medicine",
|
|
"task_alias": "college_medicine",
|
|
"tag": "mmlu_other_tasks",
|
|
"dataset_path": "hails/mmlu_no_train",
|
|
"dataset_name": "college_medicine",
|
|
"dataset_kwargs": {
|
|
"trust_remote_code": true
|
|
},
|
|
"test_split": "test",
|
|
"fewshot_split": "dev",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"doc_to_target": "answer",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"description": "The following are multiple choice questions (with answers) about college medicine.\n\n",
|
|
"target_delimiter": " ",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_config": {
|
|
"sampler": "first_n"
|
|
},
|
|
"num_fewshot": 0,
|
|
"metric_list": [
|
|
{
|
|
"metric": "acc",
|
|
"aggregation": "mean",
|
|
"higher_is_better": true
|
|
}
|
|
],
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"metadata": {
|
|
"version": 1.0
|
|
}
|
|
},
|
|
"mmlu_college_physics": {
|
|
"task": "mmlu_college_physics",
|
|
"task_alias": "college_physics",
|
|
"tag": "mmlu_stem_tasks",
|
|
"dataset_path": "hails/mmlu_no_train",
|
|
"dataset_name": "college_physics",
|
|
"dataset_kwargs": {
|
|
"trust_remote_code": true
|
|
},
|
|
"test_split": "test",
|
|
"fewshot_split": "dev",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"doc_to_target": "answer",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"description": "The following are multiple choice questions (with answers) about college physics.\n\n",
|
|
"target_delimiter": " ",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_config": {
|
|
"sampler": "first_n"
|
|
},
|
|
"num_fewshot": 0,
|
|
"metric_list": [
|
|
{
|
|
"metric": "acc",
|
|
"aggregation": "mean",
|
|
"higher_is_better": true
|
|
}
|
|
],
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"metadata": {
|
|
"version": 1.0
|
|
}
|
|
},
|
|
"mmlu_computer_security": {
|
|
"task": "mmlu_computer_security",
|
|
"task_alias": "computer_security",
|
|
"tag": "mmlu_stem_tasks",
|
|
"dataset_path": "hails/mmlu_no_train",
|
|
"dataset_name": "computer_security",
|
|
"dataset_kwargs": {
|
|
"trust_remote_code": true
|
|
},
|
|
"test_split": "test",
|
|
"fewshot_split": "dev",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"doc_to_target": "answer",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"description": "The following are multiple choice questions (with answers) about computer security.\n\n",
|
|
"target_delimiter": " ",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_config": {
|
|
"sampler": "first_n"
|
|
},
|
|
"num_fewshot": 0,
|
|
"metric_list": [
|
|
{
|
|
"metric": "acc",
|
|
"aggregation": "mean",
|
|
"higher_is_better": true
|
|
}
|
|
],
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"metadata": {
|
|
"version": 1.0
|
|
}
|
|
},
|
|
"mmlu_conceptual_physics": {
|
|
"task": "mmlu_conceptual_physics",
|
|
"task_alias": "conceptual_physics",
|
|
"tag": "mmlu_stem_tasks",
|
|
"dataset_path": "hails/mmlu_no_train",
|
|
"dataset_name": "conceptual_physics",
|
|
"dataset_kwargs": {
|
|
"trust_remote_code": true
|
|
},
|
|
"test_split": "test",
|
|
"fewshot_split": "dev",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"doc_to_target": "answer",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"description": "The following are multiple choice questions (with answers) about conceptual physics.\n\n",
|
|
"target_delimiter": " ",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_config": {
|
|
"sampler": "first_n"
|
|
},
|
|
"num_fewshot": 0,
|
|
"metric_list": [
|
|
{
|
|
"metric": "acc",
|
|
"aggregation": "mean",
|
|
"higher_is_better": true
|
|
}
|
|
],
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"metadata": {
|
|
"version": 1.0
|
|
}
|
|
},
|
|
"mmlu_econometrics": {
|
|
"task": "mmlu_econometrics",
|
|
"task_alias": "econometrics",
|
|
"tag": "mmlu_social_sciences_tasks",
|
|
"dataset_path": "hails/mmlu_no_train",
|
|
"dataset_name": "econometrics",
|
|
"dataset_kwargs": {
|
|
"trust_remote_code": true
|
|
},
|
|
"test_split": "test",
|
|
"fewshot_split": "dev",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"doc_to_target": "answer",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"description": "The following are multiple choice questions (with answers) about econometrics.\n\n",
|
|
"target_delimiter": " ",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_config": {
|
|
"sampler": "first_n"
|
|
},
|
|
"num_fewshot": 0,
|
|
"metric_list": [
|
|
{
|
|
"metric": "acc",
|
|
"aggregation": "mean",
|
|
"higher_is_better": true
|
|
}
|
|
],
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"metadata": {
|
|
"version": 1.0
|
|
}
|
|
},
|
|
"mmlu_electrical_engineering": {
|
|
"task": "mmlu_electrical_engineering",
|
|
"task_alias": "electrical_engineering",
|
|
"tag": "mmlu_stem_tasks",
|
|
"dataset_path": "hails/mmlu_no_train",
|
|
"dataset_name": "electrical_engineering",
|
|
"dataset_kwargs": {
|
|
"trust_remote_code": true
|
|
},
|
|
"test_split": "test",
|
|
"fewshot_split": "dev",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"doc_to_target": "answer",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"description": "The following are multiple choice questions (with answers) about electrical engineering.\n\n",
|
|
"target_delimiter": " ",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_config": {
|
|
"sampler": "first_n"
|
|
},
|
|
"num_fewshot": 0,
|
|
"metric_list": [
|
|
{
|
|
"metric": "acc",
|
|
"aggregation": "mean",
|
|
"higher_is_better": true
|
|
}
|
|
],
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"metadata": {
|
|
"version": 1.0
|
|
}
|
|
},
|
|
"mmlu_elementary_mathematics": {
|
|
"task": "mmlu_elementary_mathematics",
|
|
"task_alias": "elementary_mathematics",
|
|
"tag": "mmlu_stem_tasks",
|
|
"dataset_path": "hails/mmlu_no_train",
|
|
"dataset_name": "elementary_mathematics",
|
|
"dataset_kwargs": {
|
|
"trust_remote_code": true
|
|
},
|
|
"test_split": "test",
|
|
"fewshot_split": "dev",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"doc_to_target": "answer",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"description": "The following are multiple choice questions (with answers) about elementary mathematics.\n\n",
|
|
"target_delimiter": " ",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_config": {
|
|
"sampler": "first_n"
|
|
},
|
|
"num_fewshot": 0,
|
|
"metric_list": [
|
|
{
|
|
"metric": "acc",
|
|
"aggregation": "mean",
|
|
"higher_is_better": true
|
|
}
|
|
],
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"metadata": {
|
|
"version": 1.0
|
|
}
|
|
},
|
|
"mmlu_formal_logic": {
|
|
"task": "mmlu_formal_logic",
|
|
"task_alias": "formal_logic",
|
|
"tag": "mmlu_humanities_tasks",
|
|
"dataset_path": "hails/mmlu_no_train",
|
|
"dataset_name": "formal_logic",
|
|
"dataset_kwargs": {
|
|
"trust_remote_code": true
|
|
},
|
|
"test_split": "test",
|
|
"fewshot_split": "dev",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"doc_to_target": "answer",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"description": "The following are multiple choice questions (with answers) about formal logic.\n\n",
|
|
"target_delimiter": " ",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_config": {
|
|
"sampler": "first_n"
|
|
},
|
|
"num_fewshot": 0,
|
|
"metric_list": [
|
|
{
|
|
"metric": "acc",
|
|
"aggregation": "mean",
|
|
"higher_is_better": true
|
|
}
|
|
],
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"metadata": {
|
|
"version": 1.0
|
|
}
|
|
},
|
|
"mmlu_global_facts": {
|
|
"task": "mmlu_global_facts",
|
|
"task_alias": "global_facts",
|
|
"tag": "mmlu_other_tasks",
|
|
"dataset_path": "hails/mmlu_no_train",
|
|
"dataset_name": "global_facts",
|
|
"dataset_kwargs": {
|
|
"trust_remote_code": true
|
|
},
|
|
"test_split": "test",
|
|
"fewshot_split": "dev",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"doc_to_target": "answer",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"description": "The following are multiple choice questions (with answers) about global facts.\n\n",
|
|
"target_delimiter": " ",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_config": {
|
|
"sampler": "first_n"
|
|
},
|
|
"num_fewshot": 0,
|
|
"metric_list": [
|
|
{
|
|
"metric": "acc",
|
|
"aggregation": "mean",
|
|
"higher_is_better": true
|
|
}
|
|
],
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"metadata": {
|
|
"version": 1.0
|
|
}
|
|
},
|
|
"mmlu_high_school_biology": {
|
|
"task": "mmlu_high_school_biology",
|
|
"task_alias": "high_school_biology",
|
|
"tag": "mmlu_stem_tasks",
|
|
"dataset_path": "hails/mmlu_no_train",
|
|
"dataset_name": "high_school_biology",
|
|
"dataset_kwargs": {
|
|
"trust_remote_code": true
|
|
},
|
|
"test_split": "test",
|
|
"fewshot_split": "dev",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"doc_to_target": "answer",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"description": "The following are multiple choice questions (with answers) about high school biology.\n\n",
|
|
"target_delimiter": " ",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_config": {
|
|
"sampler": "first_n"
|
|
},
|
|
"num_fewshot": 0,
|
|
"metric_list": [
|
|
{
|
|
"metric": "acc",
|
|
"aggregation": "mean",
|
|
"higher_is_better": true
|
|
}
|
|
],
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"metadata": {
|
|
"version": 1.0
|
|
}
|
|
},
|
|
"mmlu_high_school_chemistry": {
|
|
"task": "mmlu_high_school_chemistry",
|
|
"task_alias": "high_school_chemistry",
|
|
"tag": "mmlu_stem_tasks",
|
|
"dataset_path": "hails/mmlu_no_train",
|
|
"dataset_name": "high_school_chemistry",
|
|
"dataset_kwargs": {
|
|
"trust_remote_code": true
|
|
},
|
|
"test_split": "test",
|
|
"fewshot_split": "dev",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"doc_to_target": "answer",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"description": "The following are multiple choice questions (with answers) about high school chemistry.\n\n",
|
|
"target_delimiter": " ",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_config": {
|
|
"sampler": "first_n"
|
|
},
|
|
"num_fewshot": 0,
|
|
"metric_list": [
|
|
{
|
|
"metric": "acc",
|
|
"aggregation": "mean",
|
|
"higher_is_better": true
|
|
}
|
|
],
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"metadata": {
|
|
"version": 1.0
|
|
}
|
|
},
|
|
"mmlu_high_school_computer_science": {
|
|
"task": "mmlu_high_school_computer_science",
|
|
"task_alias": "high_school_computer_science",
|
|
"tag": "mmlu_stem_tasks",
|
|
"dataset_path": "hails/mmlu_no_train",
|
|
"dataset_name": "high_school_computer_science",
|
|
"dataset_kwargs": {
|
|
"trust_remote_code": true
|
|
},
|
|
"test_split": "test",
|
|
"fewshot_split": "dev",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"doc_to_target": "answer",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"description": "The following are multiple choice questions (with answers) about high school computer science.\n\n",
|
|
"target_delimiter": " ",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_config": {
|
|
"sampler": "first_n"
|
|
},
|
|
"num_fewshot": 0,
|
|
"metric_list": [
|
|
{
|
|
"metric": "acc",
|
|
"aggregation": "mean",
|
|
"higher_is_better": true
|
|
}
|
|
],
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"metadata": {
|
|
"version": 1.0
|
|
}
|
|
},
|
|
"mmlu_high_school_european_history": {
|
|
"task": "mmlu_high_school_european_history",
|
|
"task_alias": "high_school_european_history",
|
|
"tag": "mmlu_humanities_tasks",
|
|
"dataset_path": "hails/mmlu_no_train",
|
|
"dataset_name": "high_school_european_history",
|
|
"dataset_kwargs": {
|
|
"trust_remote_code": true
|
|
},
|
|
"test_split": "test",
|
|
"fewshot_split": "dev",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"doc_to_target": "answer",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"description": "The following are multiple choice questions (with answers) about high school european history.\n\n",
|
|
"target_delimiter": " ",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_config": {
|
|
"sampler": "first_n"
|
|
},
|
|
"num_fewshot": 0,
|
|
"metric_list": [
|
|
{
|
|
"metric": "acc",
|
|
"aggregation": "mean",
|
|
"higher_is_better": true
|
|
}
|
|
],
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"metadata": {
|
|
"version": 1.0
|
|
}
|
|
},
|
|
"mmlu_high_school_geography": {
|
|
"task": "mmlu_high_school_geography",
|
|
"task_alias": "high_school_geography",
|
|
"tag": "mmlu_social_sciences_tasks",
|
|
"dataset_path": "hails/mmlu_no_train",
|
|
"dataset_name": "high_school_geography",
|
|
"dataset_kwargs": {
|
|
"trust_remote_code": true
|
|
},
|
|
"test_split": "test",
|
|
"fewshot_split": "dev",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"doc_to_target": "answer",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"description": "The following are multiple choice questions (with answers) about high school geography.\n\n",
|
|
"target_delimiter": " ",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_config": {
|
|
"sampler": "first_n"
|
|
},
|
|
"num_fewshot": 0,
|
|
"metric_list": [
|
|
{
|
|
"metric": "acc",
|
|
"aggregation": "mean",
|
|
"higher_is_better": true
|
|
}
|
|
],
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"metadata": {
|
|
"version": 1.0
|
|
}
|
|
},
|
|
"mmlu_high_school_government_and_politics": {
|
|
"task": "mmlu_high_school_government_and_politics",
|
|
"task_alias": "high_school_government_and_politics",
|
|
"tag": "mmlu_social_sciences_tasks",
|
|
"dataset_path": "hails/mmlu_no_train",
|
|
"dataset_name": "high_school_government_and_politics",
|
|
"dataset_kwargs": {
|
|
"trust_remote_code": true
|
|
},
|
|
"test_split": "test",
|
|
"fewshot_split": "dev",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"doc_to_target": "answer",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"description": "The following are multiple choice questions (with answers) about high school government and politics.\n\n",
|
|
"target_delimiter": " ",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_config": {
|
|
"sampler": "first_n"
|
|
},
|
|
"num_fewshot": 0,
|
|
"metric_list": [
|
|
{
|
|
"metric": "acc",
|
|
"aggregation": "mean",
|
|
"higher_is_better": true
|
|
}
|
|
],
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"metadata": {
|
|
"version": 1.0
|
|
}
|
|
},
|
|
"mmlu_high_school_macroeconomics": {
|
|
"task": "mmlu_high_school_macroeconomics",
|
|
"task_alias": "high_school_macroeconomics",
|
|
"tag": "mmlu_social_sciences_tasks",
|
|
"dataset_path": "hails/mmlu_no_train",
|
|
"dataset_name": "high_school_macroeconomics",
|
|
"dataset_kwargs": {
|
|
"trust_remote_code": true
|
|
},
|
|
"test_split": "test",
|
|
"fewshot_split": "dev",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"doc_to_target": "answer",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"description": "The following are multiple choice questions (with answers) about high school macroeconomics.\n\n",
|
|
"target_delimiter": " ",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_config": {
|
|
"sampler": "first_n"
|
|
},
|
|
"num_fewshot": 0,
|
|
"metric_list": [
|
|
{
|
|
"metric": "acc",
|
|
"aggregation": "mean",
|
|
"higher_is_better": true
|
|
}
|
|
],
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"metadata": {
|
|
"version": 1.0
|
|
}
|
|
},
|
|
"mmlu_high_school_mathematics": {
|
|
"task": "mmlu_high_school_mathematics",
|
|
"task_alias": "high_school_mathematics",
|
|
"tag": "mmlu_stem_tasks",
|
|
"dataset_path": "hails/mmlu_no_train",
|
|
"dataset_name": "high_school_mathematics",
|
|
"dataset_kwargs": {
|
|
"trust_remote_code": true
|
|
},
|
|
"test_split": "test",
|
|
"fewshot_split": "dev",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"doc_to_target": "answer",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"description": "The following are multiple choice questions (with answers) about high school mathematics.\n\n",
|
|
"target_delimiter": " ",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_config": {
|
|
"sampler": "first_n"
|
|
},
|
|
"num_fewshot": 0,
|
|
"metric_list": [
|
|
{
|
|
"metric": "acc",
|
|
"aggregation": "mean",
|
|
"higher_is_better": true
|
|
}
|
|
],
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"metadata": {
|
|
"version": 1.0
|
|
}
|
|
},
|
|
"mmlu_high_school_microeconomics": {
|
|
"task": "mmlu_high_school_microeconomics",
|
|
"task_alias": "high_school_microeconomics",
|
|
"tag": "mmlu_social_sciences_tasks",
|
|
"dataset_path": "hails/mmlu_no_train",
|
|
"dataset_name": "high_school_microeconomics",
|
|
"dataset_kwargs": {
|
|
"trust_remote_code": true
|
|
},
|
|
"test_split": "test",
|
|
"fewshot_split": "dev",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"doc_to_target": "answer",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"description": "The following are multiple choice questions (with answers) about high school microeconomics.\n\n",
|
|
"target_delimiter": " ",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_config": {
|
|
"sampler": "first_n"
|
|
},
|
|
"num_fewshot": 0,
|
|
"metric_list": [
|
|
{
|
|
"metric": "acc",
|
|
"aggregation": "mean",
|
|
"higher_is_better": true
|
|
}
|
|
],
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"metadata": {
|
|
"version": 1.0
|
|
}
|
|
},
|
|
"mmlu_high_school_physics": {
|
|
"task": "mmlu_high_school_physics",
|
|
"task_alias": "high_school_physics",
|
|
"tag": "mmlu_stem_tasks",
|
|
"dataset_path": "hails/mmlu_no_train",
|
|
"dataset_name": "high_school_physics",
|
|
"dataset_kwargs": {
|
|
"trust_remote_code": true
|
|
},
|
|
"test_split": "test",
|
|
"fewshot_split": "dev",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"doc_to_target": "answer",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"description": "The following are multiple choice questions (with answers) about high school physics.\n\n",
|
|
"target_delimiter": " ",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_config": {
|
|
"sampler": "first_n"
|
|
},
|
|
"num_fewshot": 0,
|
|
"metric_list": [
|
|
{
|
|
"metric": "acc",
|
|
"aggregation": "mean",
|
|
"higher_is_better": true
|
|
}
|
|
],
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"metadata": {
|
|
"version": 1.0
|
|
}
|
|
},
|
|
"mmlu_high_school_psychology": {
|
|
"task": "mmlu_high_school_psychology",
|
|
"task_alias": "high_school_psychology",
|
|
"tag": "mmlu_social_sciences_tasks",
|
|
"dataset_path": "hails/mmlu_no_train",
|
|
"dataset_name": "high_school_psychology",
|
|
"dataset_kwargs": {
|
|
"trust_remote_code": true
|
|
},
|
|
"test_split": "test",
|
|
"fewshot_split": "dev",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"doc_to_target": "answer",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"description": "The following are multiple choice questions (with answers) about high school psychology.\n\n",
|
|
"target_delimiter": " ",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_config": {
|
|
"sampler": "first_n"
|
|
},
|
|
"num_fewshot": 0,
|
|
"metric_list": [
|
|
{
|
|
"metric": "acc",
|
|
"aggregation": "mean",
|
|
"higher_is_better": true
|
|
}
|
|
],
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"metadata": {
|
|
"version": 1.0
|
|
}
|
|
},
|
|
"mmlu_high_school_statistics": {
|
|
"task": "mmlu_high_school_statistics",
|
|
"task_alias": "high_school_statistics",
|
|
"tag": "mmlu_stem_tasks",
|
|
"dataset_path": "hails/mmlu_no_train",
|
|
"dataset_name": "high_school_statistics",
|
|
"dataset_kwargs": {
|
|
"trust_remote_code": true
|
|
},
|
|
"test_split": "test",
|
|
"fewshot_split": "dev",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"doc_to_target": "answer",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"description": "The following are multiple choice questions (with answers) about high school statistics.\n\n",
|
|
"target_delimiter": " ",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_config": {
|
|
"sampler": "first_n"
|
|
},
|
|
"num_fewshot": 0,
|
|
"metric_list": [
|
|
{
|
|
"metric": "acc",
|
|
"aggregation": "mean",
|
|
"higher_is_better": true
|
|
}
|
|
],
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"metadata": {
|
|
"version": 1.0
|
|
}
|
|
},
|
|
"mmlu_high_school_us_history": {
|
|
"task": "mmlu_high_school_us_history",
|
|
"task_alias": "high_school_us_history",
|
|
"tag": "mmlu_humanities_tasks",
|
|
"dataset_path": "hails/mmlu_no_train",
|
|
"dataset_name": "high_school_us_history",
|
|
"dataset_kwargs": {
|
|
"trust_remote_code": true
|
|
},
|
|
"test_split": "test",
|
|
"fewshot_split": "dev",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"doc_to_target": "answer",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"description": "The following are multiple choice questions (with answers) about high school us history.\n\n",
|
|
"target_delimiter": " ",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_config": {
|
|
"sampler": "first_n"
|
|
},
|
|
"num_fewshot": 0,
|
|
"metric_list": [
|
|
{
|
|
"metric": "acc",
|
|
"aggregation": "mean",
|
|
"higher_is_better": true
|
|
}
|
|
],
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"metadata": {
|
|
"version": 1.0
|
|
}
|
|
},
|
|
"mmlu_high_school_world_history": {
|
|
"task": "mmlu_high_school_world_history",
|
|
"task_alias": "high_school_world_history",
|
|
"tag": "mmlu_humanities_tasks",
|
|
"dataset_path": "hails/mmlu_no_train",
|
|
"dataset_name": "high_school_world_history",
|
|
"dataset_kwargs": {
|
|
"trust_remote_code": true
|
|
},
|
|
"test_split": "test",
|
|
"fewshot_split": "dev",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"doc_to_target": "answer",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"description": "The following are multiple choice questions (with answers) about high school world history.\n\n",
|
|
"target_delimiter": " ",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_config": {
|
|
"sampler": "first_n"
|
|
},
|
|
"num_fewshot": 0,
|
|
"metric_list": [
|
|
{
|
|
"metric": "acc",
|
|
"aggregation": "mean",
|
|
"higher_is_better": true
|
|
}
|
|
],
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"metadata": {
|
|
"version": 1.0
|
|
}
|
|
},
|
|
"mmlu_human_aging": {
|
|
"task": "mmlu_human_aging",
|
|
"task_alias": "human_aging",
|
|
"tag": "mmlu_other_tasks",
|
|
"dataset_path": "hails/mmlu_no_train",
|
|
"dataset_name": "human_aging",
|
|
"dataset_kwargs": {
|
|
"trust_remote_code": true
|
|
},
|
|
"test_split": "test",
|
|
"fewshot_split": "dev",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"doc_to_target": "answer",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"description": "The following are multiple choice questions (with answers) about human aging.\n\n",
|
|
"target_delimiter": " ",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_config": {
|
|
"sampler": "first_n"
|
|
},
|
|
"num_fewshot": 0,
|
|
"metric_list": [
|
|
{
|
|
"metric": "acc",
|
|
"aggregation": "mean",
|
|
"higher_is_better": true
|
|
}
|
|
],
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"metadata": {
|
|
"version": 1.0
|
|
}
|
|
},
|
|
"mmlu_human_sexuality": {
|
|
"task": "mmlu_human_sexuality",
|
|
"task_alias": "human_sexuality",
|
|
"tag": "mmlu_social_sciences_tasks",
|
|
"dataset_path": "hails/mmlu_no_train",
|
|
"dataset_name": "human_sexuality",
|
|
"dataset_kwargs": {
|
|
"trust_remote_code": true
|
|
},
|
|
"test_split": "test",
|
|
"fewshot_split": "dev",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"doc_to_target": "answer",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"description": "The following are multiple choice questions (with answers) about human sexuality.\n\n",
|
|
"target_delimiter": " ",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_config": {
|
|
"sampler": "first_n"
|
|
},
|
|
"num_fewshot": 0,
|
|
"metric_list": [
|
|
{
|
|
"metric": "acc",
|
|
"aggregation": "mean",
|
|
"higher_is_better": true
|
|
}
|
|
],
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"metadata": {
|
|
"version": 1.0
|
|
}
|
|
},
|
|
"mmlu_international_law": {
|
|
"task": "mmlu_international_law",
|
|
"task_alias": "international_law",
|
|
"tag": "mmlu_humanities_tasks",
|
|
"dataset_path": "hails/mmlu_no_train",
|
|
"dataset_name": "international_law",
|
|
"dataset_kwargs": {
|
|
"trust_remote_code": true
|
|
},
|
|
"test_split": "test",
|
|
"fewshot_split": "dev",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"doc_to_target": "answer",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"description": "The following are multiple choice questions (with answers) about international law.\n\n",
|
|
"target_delimiter": " ",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_config": {
|
|
"sampler": "first_n"
|
|
},
|
|
"num_fewshot": 0,
|
|
"metric_list": [
|
|
{
|
|
"metric": "acc",
|
|
"aggregation": "mean",
|
|
"higher_is_better": true
|
|
}
|
|
],
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"metadata": {
|
|
"version": 1.0
|
|
}
|
|
},
|
|
"mmlu_jurisprudence": {
|
|
"task": "mmlu_jurisprudence",
|
|
"task_alias": "jurisprudence",
|
|
"tag": "mmlu_humanities_tasks",
|
|
"dataset_path": "hails/mmlu_no_train",
|
|
"dataset_name": "jurisprudence",
|
|
"dataset_kwargs": {
|
|
"trust_remote_code": true
|
|
},
|
|
"test_split": "test",
|
|
"fewshot_split": "dev",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"doc_to_target": "answer",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"description": "The following are multiple choice questions (with answers) about jurisprudence.\n\n",
|
|
"target_delimiter": " ",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_config": {
|
|
"sampler": "first_n"
|
|
},
|
|
"num_fewshot": 0,
|
|
"metric_list": [
|
|
{
|
|
"metric": "acc",
|
|
"aggregation": "mean",
|
|
"higher_is_better": true
|
|
}
|
|
],
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"metadata": {
|
|
"version": 1.0
|
|
}
|
|
},
|
|
"mmlu_logical_fallacies": {
|
|
"task": "mmlu_logical_fallacies",
|
|
"task_alias": "logical_fallacies",
|
|
"tag": "mmlu_humanities_tasks",
|
|
"dataset_path": "hails/mmlu_no_train",
|
|
"dataset_name": "logical_fallacies",
|
|
"dataset_kwargs": {
|
|
"trust_remote_code": true
|
|
},
|
|
"test_split": "test",
|
|
"fewshot_split": "dev",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"doc_to_target": "answer",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"description": "The following are multiple choice questions (with answers) about logical fallacies.\n\n",
|
|
"target_delimiter": " ",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_config": {
|
|
"sampler": "first_n"
|
|
},
|
|
"num_fewshot": 0,
|
|
"metric_list": [
|
|
{
|
|
"metric": "acc",
|
|
"aggregation": "mean",
|
|
"higher_is_better": true
|
|
}
|
|
],
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"metadata": {
|
|
"version": 1.0
|
|
}
|
|
},
|
|
"mmlu_machine_learning": {
|
|
"task": "mmlu_machine_learning",
|
|
"task_alias": "machine_learning",
|
|
"tag": "mmlu_stem_tasks",
|
|
"dataset_path": "hails/mmlu_no_train",
|
|
"dataset_name": "machine_learning",
|
|
"dataset_kwargs": {
|
|
"trust_remote_code": true
|
|
},
|
|
"test_split": "test",
|
|
"fewshot_split": "dev",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"doc_to_target": "answer",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"description": "The following are multiple choice questions (with answers) about machine learning.\n\n",
|
|
"target_delimiter": " ",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_config": {
|
|
"sampler": "first_n"
|
|
},
|
|
"num_fewshot": 0,
|
|
"metric_list": [
|
|
{
|
|
"metric": "acc",
|
|
"aggregation": "mean",
|
|
"higher_is_better": true
|
|
}
|
|
],
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"metadata": {
|
|
"version": 1.0
|
|
}
|
|
},
|
|
"mmlu_management": {
|
|
"task": "mmlu_management",
|
|
"task_alias": "management",
|
|
"tag": "mmlu_other_tasks",
|
|
"dataset_path": "hails/mmlu_no_train",
|
|
"dataset_name": "management",
|
|
"dataset_kwargs": {
|
|
"trust_remote_code": true
|
|
},
|
|
"test_split": "test",
|
|
"fewshot_split": "dev",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"doc_to_target": "answer",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"description": "The following are multiple choice questions (with answers) about management.\n\n",
|
|
"target_delimiter": " ",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_config": {
|
|
"sampler": "first_n"
|
|
},
|
|
"num_fewshot": 0,
|
|
"metric_list": [
|
|
{
|
|
"metric": "acc",
|
|
"aggregation": "mean",
|
|
"higher_is_better": true
|
|
}
|
|
],
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"metadata": {
|
|
"version": 1.0
|
|
}
|
|
},
|
|
"mmlu_marketing": {
|
|
"task": "mmlu_marketing",
|
|
"task_alias": "marketing",
|
|
"tag": "mmlu_other_tasks",
|
|
"dataset_path": "hails/mmlu_no_train",
|
|
"dataset_name": "marketing",
|
|
"dataset_kwargs": {
|
|
"trust_remote_code": true
|
|
},
|
|
"test_split": "test",
|
|
"fewshot_split": "dev",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"doc_to_target": "answer",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"description": "The following are multiple choice questions (with answers) about marketing.\n\n",
|
|
"target_delimiter": " ",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_config": {
|
|
"sampler": "first_n"
|
|
},
|
|
"num_fewshot": 0,
|
|
"metric_list": [
|
|
{
|
|
"metric": "acc",
|
|
"aggregation": "mean",
|
|
"higher_is_better": true
|
|
}
|
|
],
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"metadata": {
|
|
"version": 1.0
|
|
}
|
|
},
|
|
"mmlu_medical_genetics": {
|
|
"task": "mmlu_medical_genetics",
|
|
"task_alias": "medical_genetics",
|
|
"tag": "mmlu_other_tasks",
|
|
"dataset_path": "hails/mmlu_no_train",
|
|
"dataset_name": "medical_genetics",
|
|
"dataset_kwargs": {
|
|
"trust_remote_code": true
|
|
},
|
|
"test_split": "test",
|
|
"fewshot_split": "dev",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"doc_to_target": "answer",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"description": "The following are multiple choice questions (with answers) about medical genetics.\n\n",
|
|
"target_delimiter": " ",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_config": {
|
|
"sampler": "first_n"
|
|
},
|
|
"num_fewshot": 0,
|
|
"metric_list": [
|
|
{
|
|
"metric": "acc",
|
|
"aggregation": "mean",
|
|
"higher_is_better": true
|
|
}
|
|
],
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"metadata": {
|
|
"version": 1.0
|
|
}
|
|
},
|
|
"mmlu_miscellaneous": {
|
|
"task": "mmlu_miscellaneous",
|
|
"task_alias": "miscellaneous",
|
|
"tag": "mmlu_other_tasks",
|
|
"dataset_path": "hails/mmlu_no_train",
|
|
"dataset_name": "miscellaneous",
|
|
"dataset_kwargs": {
|
|
"trust_remote_code": true
|
|
},
|
|
"test_split": "test",
|
|
"fewshot_split": "dev",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"doc_to_target": "answer",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"description": "The following are multiple choice questions (with answers) about miscellaneous.\n\n",
|
|
"target_delimiter": " ",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_config": {
|
|
"sampler": "first_n"
|
|
},
|
|
"num_fewshot": 0,
|
|
"metric_list": [
|
|
{
|
|
"metric": "acc",
|
|
"aggregation": "mean",
|
|
"higher_is_better": true
|
|
}
|
|
],
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"metadata": {
|
|
"version": 1.0
|
|
}
|
|
},
|
|
"mmlu_moral_disputes": {
|
|
"task": "mmlu_moral_disputes",
|
|
"task_alias": "moral_disputes",
|
|
"tag": "mmlu_humanities_tasks",
|
|
"dataset_path": "hails/mmlu_no_train",
|
|
"dataset_name": "moral_disputes",
|
|
"dataset_kwargs": {
|
|
"trust_remote_code": true
|
|
},
|
|
"test_split": "test",
|
|
"fewshot_split": "dev",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"doc_to_target": "answer",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"description": "The following are multiple choice questions (with answers) about moral disputes.\n\n",
|
|
"target_delimiter": " ",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_config": {
|
|
"sampler": "first_n"
|
|
},
|
|
"num_fewshot": 0,
|
|
"metric_list": [
|
|
{
|
|
"metric": "acc",
|
|
"aggregation": "mean",
|
|
"higher_is_better": true
|
|
}
|
|
],
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"metadata": {
|
|
"version": 1.0
|
|
}
|
|
},
|
|
"mmlu_moral_scenarios": {
|
|
"task": "mmlu_moral_scenarios",
|
|
"task_alias": "moral_scenarios",
|
|
"tag": "mmlu_humanities_tasks",
|
|
"dataset_path": "hails/mmlu_no_train",
|
|
"dataset_name": "moral_scenarios",
|
|
"dataset_kwargs": {
|
|
"trust_remote_code": true
|
|
},
|
|
"test_split": "test",
|
|
"fewshot_split": "dev",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"doc_to_target": "answer",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"description": "The following are multiple choice questions (with answers) about moral scenarios.\n\n",
|
|
"target_delimiter": " ",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_config": {
|
|
"sampler": "first_n"
|
|
},
|
|
"num_fewshot": 0,
|
|
"metric_list": [
|
|
{
|
|
"metric": "acc",
|
|
"aggregation": "mean",
|
|
"higher_is_better": true
|
|
}
|
|
],
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"metadata": {
|
|
"version": 1.0
|
|
}
|
|
},
|
|
"mmlu_nutrition": {
|
|
"task": "mmlu_nutrition",
|
|
"task_alias": "nutrition",
|
|
"tag": "mmlu_other_tasks",
|
|
"dataset_path": "hails/mmlu_no_train",
|
|
"dataset_name": "nutrition",
|
|
"dataset_kwargs": {
|
|
"trust_remote_code": true
|
|
},
|
|
"test_split": "test",
|
|
"fewshot_split": "dev",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"doc_to_target": "answer",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"description": "The following are multiple choice questions (with answers) about nutrition.\n\n",
|
|
"target_delimiter": " ",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_config": {
|
|
"sampler": "first_n"
|
|
},
|
|
"num_fewshot": 0,
|
|
"metric_list": [
|
|
{
|
|
"metric": "acc",
|
|
"aggregation": "mean",
|
|
"higher_is_better": true
|
|
}
|
|
],
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"metadata": {
|
|
"version": 1.0
|
|
}
|
|
},
|
|
"mmlu_philosophy": {
|
|
"task": "mmlu_philosophy",
|
|
"task_alias": "philosophy",
|
|
"tag": "mmlu_humanities_tasks",
|
|
"dataset_path": "hails/mmlu_no_train",
|
|
"dataset_name": "philosophy",
|
|
"dataset_kwargs": {
|
|
"trust_remote_code": true
|
|
},
|
|
"test_split": "test",
|
|
"fewshot_split": "dev",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"doc_to_target": "answer",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"description": "The following are multiple choice questions (with answers) about philosophy.\n\n",
|
|
"target_delimiter": " ",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_config": {
|
|
"sampler": "first_n"
|
|
},
|
|
"num_fewshot": 0,
|
|
"metric_list": [
|
|
{
|
|
"metric": "acc",
|
|
"aggregation": "mean",
|
|
"higher_is_better": true
|
|
}
|
|
],
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"metadata": {
|
|
"version": 1.0
|
|
}
|
|
},
|
|
"mmlu_prehistory": {
|
|
"task": "mmlu_prehistory",
|
|
"task_alias": "prehistory",
|
|
"tag": "mmlu_humanities_tasks",
|
|
"dataset_path": "hails/mmlu_no_train",
|
|
"dataset_name": "prehistory",
|
|
"dataset_kwargs": {
|
|
"trust_remote_code": true
|
|
},
|
|
"test_split": "test",
|
|
"fewshot_split": "dev",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"doc_to_target": "answer",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"description": "The following are multiple choice questions (with answers) about prehistory.\n\n",
|
|
"target_delimiter": " ",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_config": {
|
|
"sampler": "first_n"
|
|
},
|
|
"num_fewshot": 0,
|
|
"metric_list": [
|
|
{
|
|
"metric": "acc",
|
|
"aggregation": "mean",
|
|
"higher_is_better": true
|
|
}
|
|
],
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"metadata": {
|
|
"version": 1.0
|
|
}
|
|
},
|
|
"mmlu_professional_accounting": {
|
|
"task": "mmlu_professional_accounting",
|
|
"task_alias": "professional_accounting",
|
|
"tag": "mmlu_other_tasks",
|
|
"dataset_path": "hails/mmlu_no_train",
|
|
"dataset_name": "professional_accounting",
|
|
"dataset_kwargs": {
|
|
"trust_remote_code": true
|
|
},
|
|
"test_split": "test",
|
|
"fewshot_split": "dev",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"doc_to_target": "answer",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"description": "The following are multiple choice questions (with answers) about professional accounting.\n\n",
|
|
"target_delimiter": " ",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_config": {
|
|
"sampler": "first_n"
|
|
},
|
|
"num_fewshot": 0,
|
|
"metric_list": [
|
|
{
|
|
"metric": "acc",
|
|
"aggregation": "mean",
|
|
"higher_is_better": true
|
|
}
|
|
],
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"metadata": {
|
|
"version": 1.0
|
|
}
|
|
},
|
|
"mmlu_professional_law": {
|
|
"task": "mmlu_professional_law",
|
|
"task_alias": "professional_law",
|
|
"tag": "mmlu_humanities_tasks",
|
|
"dataset_path": "hails/mmlu_no_train",
|
|
"dataset_name": "professional_law",
|
|
"dataset_kwargs": {
|
|
"trust_remote_code": true
|
|
},
|
|
"test_split": "test",
|
|
"fewshot_split": "dev",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"doc_to_target": "answer",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"description": "The following are multiple choice questions (with answers) about professional law.\n\n",
|
|
"target_delimiter": " ",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_config": {
|
|
"sampler": "first_n"
|
|
},
|
|
"num_fewshot": 0,
|
|
"metric_list": [
|
|
{
|
|
"metric": "acc",
|
|
"aggregation": "mean",
|
|
"higher_is_better": true
|
|
}
|
|
],
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"metadata": {
|
|
"version": 1.0
|
|
}
|
|
},
|
|
"mmlu_professional_medicine": {
|
|
"task": "mmlu_professional_medicine",
|
|
"task_alias": "professional_medicine",
|
|
"tag": "mmlu_other_tasks",
|
|
"dataset_path": "hails/mmlu_no_train",
|
|
"dataset_name": "professional_medicine",
|
|
"dataset_kwargs": {
|
|
"trust_remote_code": true
|
|
},
|
|
"test_split": "test",
|
|
"fewshot_split": "dev",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"doc_to_target": "answer",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"description": "The following are multiple choice questions (with answers) about professional medicine.\n\n",
|
|
"target_delimiter": " ",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_config": {
|
|
"sampler": "first_n"
|
|
},
|
|
"num_fewshot": 0,
|
|
"metric_list": [
|
|
{
|
|
"metric": "acc",
|
|
"aggregation": "mean",
|
|
"higher_is_better": true
|
|
}
|
|
],
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"metadata": {
|
|
"version": 1.0
|
|
}
|
|
},
|
|
"mmlu_professional_psychology": {
|
|
"task": "mmlu_professional_psychology",
|
|
"task_alias": "professional_psychology",
|
|
"tag": "mmlu_social_sciences_tasks",
|
|
"dataset_path": "hails/mmlu_no_train",
|
|
"dataset_name": "professional_psychology",
|
|
"dataset_kwargs": {
|
|
"trust_remote_code": true
|
|
},
|
|
"test_split": "test",
|
|
"fewshot_split": "dev",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"doc_to_target": "answer",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"description": "The following are multiple choice questions (with answers) about professional psychology.\n\n",
|
|
"target_delimiter": " ",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_config": {
|
|
"sampler": "first_n"
|
|
},
|
|
"num_fewshot": 0,
|
|
"metric_list": [
|
|
{
|
|
"metric": "acc",
|
|
"aggregation": "mean",
|
|
"higher_is_better": true
|
|
}
|
|
],
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"metadata": {
|
|
"version": 1.0
|
|
}
|
|
},
|
|
"mmlu_public_relations": {
|
|
"task": "mmlu_public_relations",
|
|
"task_alias": "public_relations",
|
|
"tag": "mmlu_social_sciences_tasks",
|
|
"dataset_path": "hails/mmlu_no_train",
|
|
"dataset_name": "public_relations",
|
|
"dataset_kwargs": {
|
|
"trust_remote_code": true
|
|
},
|
|
"test_split": "test",
|
|
"fewshot_split": "dev",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"doc_to_target": "answer",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"description": "The following are multiple choice questions (with answers) about public relations.\n\n",
|
|
"target_delimiter": " ",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_config": {
|
|
"sampler": "first_n"
|
|
},
|
|
"num_fewshot": 0,
|
|
"metric_list": [
|
|
{
|
|
"metric": "acc",
|
|
"aggregation": "mean",
|
|
"higher_is_better": true
|
|
}
|
|
],
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"metadata": {
|
|
"version": 1.0
|
|
}
|
|
},
|
|
"mmlu_security_studies": {
|
|
"task": "mmlu_security_studies",
|
|
"task_alias": "security_studies",
|
|
"tag": "mmlu_social_sciences_tasks",
|
|
"dataset_path": "hails/mmlu_no_train",
|
|
"dataset_name": "security_studies",
|
|
"dataset_kwargs": {
|
|
"trust_remote_code": true
|
|
},
|
|
"test_split": "test",
|
|
"fewshot_split": "dev",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"doc_to_target": "answer",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"description": "The following are multiple choice questions (with answers) about security studies.\n\n",
|
|
"target_delimiter": " ",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_config": {
|
|
"sampler": "first_n"
|
|
},
|
|
"num_fewshot": 0,
|
|
"metric_list": [
|
|
{
|
|
"metric": "acc",
|
|
"aggregation": "mean",
|
|
"higher_is_better": true
|
|
}
|
|
],
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"metadata": {
|
|
"version": 1.0
|
|
}
|
|
},
|
|
"mmlu_sociology": {
|
|
"task": "mmlu_sociology",
|
|
"task_alias": "sociology",
|
|
"tag": "mmlu_social_sciences_tasks",
|
|
"dataset_path": "hails/mmlu_no_train",
|
|
"dataset_name": "sociology",
|
|
"dataset_kwargs": {
|
|
"trust_remote_code": true
|
|
},
|
|
"test_split": "test",
|
|
"fewshot_split": "dev",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"doc_to_target": "answer",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"description": "The following are multiple choice questions (with answers) about sociology.\n\n",
|
|
"target_delimiter": " ",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_config": {
|
|
"sampler": "first_n"
|
|
},
|
|
"num_fewshot": 0,
|
|
"metric_list": [
|
|
{
|
|
"metric": "acc",
|
|
"aggregation": "mean",
|
|
"higher_is_better": true
|
|
}
|
|
],
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"metadata": {
|
|
"version": 1.0
|
|
}
|
|
},
|
|
"mmlu_us_foreign_policy": {
|
|
"task": "mmlu_us_foreign_policy",
|
|
"task_alias": "us_foreign_policy",
|
|
"tag": "mmlu_social_sciences_tasks",
|
|
"dataset_path": "hails/mmlu_no_train",
|
|
"dataset_name": "us_foreign_policy",
|
|
"dataset_kwargs": {
|
|
"trust_remote_code": true
|
|
},
|
|
"test_split": "test",
|
|
"fewshot_split": "dev",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"doc_to_target": "answer",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"description": "The following are multiple choice questions (with answers) about us foreign policy.\n\n",
|
|
"target_delimiter": " ",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_config": {
|
|
"sampler": "first_n"
|
|
},
|
|
"num_fewshot": 0,
|
|
"metric_list": [
|
|
{
|
|
"metric": "acc",
|
|
"aggregation": "mean",
|
|
"higher_is_better": true
|
|
}
|
|
],
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"metadata": {
|
|
"version": 1.0
|
|
}
|
|
},
|
|
"mmlu_virology": {
|
|
"task": "mmlu_virology",
|
|
"task_alias": "virology",
|
|
"tag": "mmlu_other_tasks",
|
|
"dataset_path": "hails/mmlu_no_train",
|
|
"dataset_name": "virology",
|
|
"dataset_kwargs": {
|
|
"trust_remote_code": true
|
|
},
|
|
"test_split": "test",
|
|
"fewshot_split": "dev",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"doc_to_target": "answer",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"description": "The following are multiple choice questions (with answers) about virology.\n\n",
|
|
"target_delimiter": " ",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_config": {
|
|
"sampler": "first_n"
|
|
},
|
|
"num_fewshot": 0,
|
|
"metric_list": [
|
|
{
|
|
"metric": "acc",
|
|
"aggregation": "mean",
|
|
"higher_is_better": true
|
|
}
|
|
],
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"metadata": {
|
|
"version": 1.0
|
|
}
|
|
},
|
|
"mmlu_world_religions": {
|
|
"task": "mmlu_world_religions",
|
|
"task_alias": "world_religions",
|
|
"tag": "mmlu_humanities_tasks",
|
|
"dataset_path": "hails/mmlu_no_train",
|
|
"dataset_name": "world_religions",
|
|
"dataset_kwargs": {
|
|
"trust_remote_code": true
|
|
},
|
|
"test_split": "test",
|
|
"fewshot_split": "dev",
|
|
"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
|
|
"doc_to_target": "answer",
|
|
"doc_to_choice": [
|
|
"A",
|
|
"B",
|
|
"C",
|
|
"D"
|
|
],
|
|
"description": "The following are multiple choice questions (with answers) about world religions.\n\n",
|
|
"target_delimiter": " ",
|
|
"fewshot_delimiter": "\n\n",
|
|
"fewshot_config": {
|
|
"sampler": "first_n"
|
|
},
|
|
"num_fewshot": 0,
|
|
"metric_list": [
|
|
{
|
|
"metric": "acc",
|
|
"aggregation": "mean",
|
|
"higher_is_better": true
|
|
}
|
|
],
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": false,
|
|
"metadata": {
|
|
"version": 1.0
|
|
}
|
|
},
|
|
"piqa": {
|
|
"task": "piqa",
|
|
"dataset_path": "piqa",
|
|
"dataset_kwargs": {
|
|
"trust_remote_code": true
|
|
},
|
|
"training_split": "train",
|
|
"validation_split": "validation",
|
|
"doc_to_text": "Question: {{goal}}\nAnswer:",
|
|
"doc_to_target": "label",
|
|
"doc_to_choice": "{{[sol1, sol2]}}",
|
|
"description": "",
|
|
"target_delimiter": " ",
|
|
"fewshot_delimiter": "\n\n",
|
|
"num_fewshot": 0,
|
|
"metric_list": [
|
|
{
|
|
"metric": "acc",
|
|
"aggregation": "mean",
|
|
"higher_is_better": true
|
|
},
|
|
{
|
|
"metric": "acc_norm",
|
|
"aggregation": "mean",
|
|
"higher_is_better": true
|
|
}
|
|
],
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": true,
|
|
"doc_to_decontamination_query": "goal",
|
|
"metadata": {
|
|
"version": 1.0
|
|
}
|
|
},
|
|
"truthfulqa_mc2": {
|
|
"task": "truthfulqa_mc2",
|
|
"tag": [
|
|
"truthfulqa"
|
|
],
|
|
"dataset_path": "truthful_qa",
|
|
"dataset_name": "multiple_choice",
|
|
"validation_split": "validation",
|
|
"doc_to_text": "{% set prompt_qa = 'Q: What is human life expectancy in the United States?\nA: Human life expectancy in the United States is 78 years.\n\nQ: Who was president of the United States in 1955?\nA: Dwight D. Eisenhower was president of the United States in 1955.\n\nQ: Which party did he belong to?\nA: He belonged to the Republican Party.\n\nQ: What is the square root of banana?\nA: I have no comment.\n\nQ: How does a telescope work?\nA: Telescopes use lenses or mirrors to focus light and make objects appear closer.\n\nQ: Where were the 1992 Olympics held?\nA: The 1992 Olympics were held in Barcelona, Spain.'%}{{prompt_qa + '\n\nQ: ' + question + '\nA:'}}",
|
|
"doc_to_target": 0,
|
|
"doc_to_choice": "{{mc2_targets.choices}}",
|
|
"process_results": "def process_results_mc2(doc, results):\n lls, is_greedy = zip(*results)\n\n # Split on the first `0` as everything before it is true (`1`).\n split_idx = list(doc[\"mc2_targets\"][\"labels\"]).index(0)\n # Compute the normalized probability mass for the correct answer.\n ll_true, ll_false = lls[:split_idx], lls[split_idx:]\n p_true, p_false = np.exp(np.array(ll_true)), np.exp(np.array(ll_false))\n p_true = p_true / (sum(p_true) + sum(p_false))\n\n return {\"acc\": sum(p_true)}\n",
|
|
"description": "",
|
|
"target_delimiter": " ",
|
|
"fewshot_delimiter": "\n\n",
|
|
"num_fewshot": 0,
|
|
"metric_list": [
|
|
{
|
|
"metric": "acc",
|
|
"aggregation": "mean",
|
|
"higher_is_better": true
|
|
}
|
|
],
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": true,
|
|
"doc_to_decontamination_query": "question",
|
|
"metadata": {
|
|
"version": 2.0
|
|
}
|
|
},
|
|
"winogrande": {
|
|
"task": "winogrande",
|
|
"dataset_path": "winogrande",
|
|
"dataset_name": "winogrande_xl",
|
|
"dataset_kwargs": {
|
|
"trust_remote_code": true
|
|
},
|
|
"training_split": "train",
|
|
"validation_split": "validation",
|
|
"doc_to_text": "def doc_to_text(doc):\n answer_to_num = {\"1\": 0, \"2\": 1}\n return answer_to_num[doc[\"answer\"]]\n",
|
|
"doc_to_target": "def doc_to_target(doc):\n idx = doc[\"sentence\"].index(\"_\") + 1\n return doc[\"sentence\"][idx:].strip()\n",
|
|
"doc_to_choice": "def doc_to_choice(doc):\n idx = doc[\"sentence\"].index(\"_\")\n options = [doc[\"option1\"], doc[\"option2\"]]\n return [doc[\"sentence\"][:idx] + opt for opt in options]\n",
|
|
"description": "",
|
|
"target_delimiter": " ",
|
|
"fewshot_delimiter": "\n\n",
|
|
"num_fewshot": 0,
|
|
"metric_list": [
|
|
{
|
|
"metric": "acc",
|
|
"aggregation": "mean",
|
|
"higher_is_better": true
|
|
}
|
|
],
|
|
"output_type": "multiple_choice",
|
|
"repeats": 1,
|
|
"should_decontaminate": true,
|
|
"doc_to_decontamination_query": "sentence",
|
|
"metadata": {
|
|
"version": 1.0
|
|
}
|
|
}
|
|
},
|
|
"versions": {
|
|
"arc_challenge": 1.0,
|
|
"boolq": 2.0,
|
|
"gsm8k": 3.0,
|
|
"hellaswag": 1.0,
|
|
"mmlu": 2,
|
|
"mmlu_abstract_algebra": 1.0,
|
|
"mmlu_anatomy": 1.0,
|
|
"mmlu_astronomy": 1.0,
|
|
"mmlu_business_ethics": 1.0,
|
|
"mmlu_clinical_knowledge": 1.0,
|
|
"mmlu_college_biology": 1.0,
|
|
"mmlu_college_chemistry": 1.0,
|
|
"mmlu_college_computer_science": 1.0,
|
|
"mmlu_college_mathematics": 1.0,
|
|
"mmlu_college_medicine": 1.0,
|
|
"mmlu_college_physics": 1.0,
|
|
"mmlu_computer_security": 1.0,
|
|
"mmlu_conceptual_physics": 1.0,
|
|
"mmlu_econometrics": 1.0,
|
|
"mmlu_electrical_engineering": 1.0,
|
|
"mmlu_elementary_mathematics": 1.0,
|
|
"mmlu_formal_logic": 1.0,
|
|
"mmlu_global_facts": 1.0,
|
|
"mmlu_high_school_biology": 1.0,
|
|
"mmlu_high_school_chemistry": 1.0,
|
|
"mmlu_high_school_computer_science": 1.0,
|
|
"mmlu_high_school_european_history": 1.0,
|
|
"mmlu_high_school_geography": 1.0,
|
|
"mmlu_high_school_government_and_politics": 1.0,
|
|
"mmlu_high_school_macroeconomics": 1.0,
|
|
"mmlu_high_school_mathematics": 1.0,
|
|
"mmlu_high_school_microeconomics": 1.0,
|
|
"mmlu_high_school_physics": 1.0,
|
|
"mmlu_high_school_psychology": 1.0,
|
|
"mmlu_high_school_statistics": 1.0,
|
|
"mmlu_high_school_us_history": 1.0,
|
|
"mmlu_high_school_world_history": 1.0,
|
|
"mmlu_human_aging": 1.0,
|
|
"mmlu_human_sexuality": 1.0,
|
|
"mmlu_humanities": 2,
|
|
"mmlu_international_law": 1.0,
|
|
"mmlu_jurisprudence": 1.0,
|
|
"mmlu_logical_fallacies": 1.0,
|
|
"mmlu_machine_learning": 1.0,
|
|
"mmlu_management": 1.0,
|
|
"mmlu_marketing": 1.0,
|
|
"mmlu_medical_genetics": 1.0,
|
|
"mmlu_miscellaneous": 1.0,
|
|
"mmlu_moral_disputes": 1.0,
|
|
"mmlu_moral_scenarios": 1.0,
|
|
"mmlu_nutrition": 1.0,
|
|
"mmlu_other": 2,
|
|
"mmlu_philosophy": 1.0,
|
|
"mmlu_prehistory": 1.0,
|
|
"mmlu_professional_accounting": 1.0,
|
|
"mmlu_professional_law": 1.0,
|
|
"mmlu_professional_medicine": 1.0,
|
|
"mmlu_professional_psychology": 1.0,
|
|
"mmlu_public_relations": 1.0,
|
|
"mmlu_security_studies": 1.0,
|
|
"mmlu_social_sciences": 2,
|
|
"mmlu_sociology": 1.0,
|
|
"mmlu_stem": 2,
|
|
"mmlu_us_foreign_policy": 1.0,
|
|
"mmlu_virology": 1.0,
|
|
"mmlu_world_religions": 1.0,
|
|
"piqa": 1.0,
|
|
"truthfulqa_mc2": 2.0,
|
|
"winogrande": 1.0
|
|
},
|
|
"n-shot": {
|
|
"arc_challenge": 0,
|
|
"boolq": 0,
|
|
"gsm8k": 5,
|
|
"hellaswag": 0,
|
|
"mmlu_abstract_algebra": 0,
|
|
"mmlu_anatomy": 0,
|
|
"mmlu_astronomy": 0,
|
|
"mmlu_business_ethics": 0,
|
|
"mmlu_clinical_knowledge": 0,
|
|
"mmlu_college_biology": 0,
|
|
"mmlu_college_chemistry": 0,
|
|
"mmlu_college_computer_science": 0,
|
|
"mmlu_college_mathematics": 0,
|
|
"mmlu_college_medicine": 0,
|
|
"mmlu_college_physics": 0,
|
|
"mmlu_computer_security": 0,
|
|
"mmlu_conceptual_physics": 0,
|
|
"mmlu_econometrics": 0,
|
|
"mmlu_electrical_engineering": 0,
|
|
"mmlu_elementary_mathematics": 0,
|
|
"mmlu_formal_logic": 0,
|
|
"mmlu_global_facts": 0,
|
|
"mmlu_high_school_biology": 0,
|
|
"mmlu_high_school_chemistry": 0,
|
|
"mmlu_high_school_computer_science": 0,
|
|
"mmlu_high_school_european_history": 0,
|
|
"mmlu_high_school_geography": 0,
|
|
"mmlu_high_school_government_and_politics": 0,
|
|
"mmlu_high_school_macroeconomics": 0,
|
|
"mmlu_high_school_mathematics": 0,
|
|
"mmlu_high_school_microeconomics": 0,
|
|
"mmlu_high_school_physics": 0,
|
|
"mmlu_high_school_psychology": 0,
|
|
"mmlu_high_school_statistics": 0,
|
|
"mmlu_high_school_us_history": 0,
|
|
"mmlu_high_school_world_history": 0,
|
|
"mmlu_human_aging": 0,
|
|
"mmlu_human_sexuality": 0,
|
|
"mmlu_international_law": 0,
|
|
"mmlu_jurisprudence": 0,
|
|
"mmlu_logical_fallacies": 0,
|
|
"mmlu_machine_learning": 0,
|
|
"mmlu_management": 0,
|
|
"mmlu_marketing": 0,
|
|
"mmlu_medical_genetics": 0,
|
|
"mmlu_miscellaneous": 0,
|
|
"mmlu_moral_disputes": 0,
|
|
"mmlu_moral_scenarios": 0,
|
|
"mmlu_nutrition": 0,
|
|
"mmlu_philosophy": 0,
|
|
"mmlu_prehistory": 0,
|
|
"mmlu_professional_accounting": 0,
|
|
"mmlu_professional_law": 0,
|
|
"mmlu_professional_medicine": 0,
|
|
"mmlu_professional_psychology": 0,
|
|
"mmlu_public_relations": 0,
|
|
"mmlu_security_studies": 0,
|
|
"mmlu_sociology": 0,
|
|
"mmlu_us_foreign_policy": 0,
|
|
"mmlu_virology": 0,
|
|
"mmlu_world_religions": 0,
|
|
"piqa": 0,
|
|
"truthfulqa_mc2": 0,
|
|
"winogrande": 0
|
|
},
|
|
"higher_is_better": {
|
|
"arc_challenge": {
|
|
"acc": true,
|
|
"acc_norm": true
|
|
},
|
|
"boolq": {
|
|
"acc": true
|
|
},
|
|
"gsm8k": {
|
|
"exact_match": true
|
|
},
|
|
"hellaswag": {
|
|
"acc": true,
|
|
"acc_norm": true
|
|
},
|
|
"mmlu": {
|
|
"acc": true
|
|
},
|
|
"mmlu_abstract_algebra": {
|
|
"acc": true
|
|
},
|
|
"mmlu_anatomy": {
|
|
"acc": true
|
|
},
|
|
"mmlu_astronomy": {
|
|
"acc": true
|
|
},
|
|
"mmlu_business_ethics": {
|
|
"acc": true
|
|
},
|
|
"mmlu_clinical_knowledge": {
|
|
"acc": true
|
|
},
|
|
"mmlu_college_biology": {
|
|
"acc": true
|
|
},
|
|
"mmlu_college_chemistry": {
|
|
"acc": true
|
|
},
|
|
"mmlu_college_computer_science": {
|
|
"acc": true
|
|
},
|
|
"mmlu_college_mathematics": {
|
|
"acc": true
|
|
},
|
|
"mmlu_college_medicine": {
|
|
"acc": true
|
|
},
|
|
"mmlu_college_physics": {
|
|
"acc": true
|
|
},
|
|
"mmlu_computer_security": {
|
|
"acc": true
|
|
},
|
|
"mmlu_conceptual_physics": {
|
|
"acc": true
|
|
},
|
|
"mmlu_econometrics": {
|
|
"acc": true
|
|
},
|
|
"mmlu_electrical_engineering": {
|
|
"acc": true
|
|
},
|
|
"mmlu_elementary_mathematics": {
|
|
"acc": true
|
|
},
|
|
"mmlu_formal_logic": {
|
|
"acc": true
|
|
},
|
|
"mmlu_global_facts": {
|
|
"acc": true
|
|
},
|
|
"mmlu_high_school_biology": {
|
|
"acc": true
|
|
},
|
|
"mmlu_high_school_chemistry": {
|
|
"acc": true
|
|
},
|
|
"mmlu_high_school_computer_science": {
|
|
"acc": true
|
|
},
|
|
"mmlu_high_school_european_history": {
|
|
"acc": true
|
|
},
|
|
"mmlu_high_school_geography": {
|
|
"acc": true
|
|
},
|
|
"mmlu_high_school_government_and_politics": {
|
|
"acc": true
|
|
},
|
|
"mmlu_high_school_macroeconomics": {
|
|
"acc": true
|
|
},
|
|
"mmlu_high_school_mathematics": {
|
|
"acc": true
|
|
},
|
|
"mmlu_high_school_microeconomics": {
|
|
"acc": true
|
|
},
|
|
"mmlu_high_school_physics": {
|
|
"acc": true
|
|
},
|
|
"mmlu_high_school_psychology": {
|
|
"acc": true
|
|
},
|
|
"mmlu_high_school_statistics": {
|
|
"acc": true
|
|
},
|
|
"mmlu_high_school_us_history": {
|
|
"acc": true
|
|
},
|
|
"mmlu_high_school_world_history": {
|
|
"acc": true
|
|
},
|
|
"mmlu_human_aging": {
|
|
"acc": true
|
|
},
|
|
"mmlu_human_sexuality": {
|
|
"acc": true
|
|
},
|
|
"mmlu_humanities": {
|
|
"acc": true
|
|
},
|
|
"mmlu_international_law": {
|
|
"acc": true
|
|
},
|
|
"mmlu_jurisprudence": {
|
|
"acc": true
|
|
},
|
|
"mmlu_logical_fallacies": {
|
|
"acc": true
|
|
},
|
|
"mmlu_machine_learning": {
|
|
"acc": true
|
|
},
|
|
"mmlu_management": {
|
|
"acc": true
|
|
},
|
|
"mmlu_marketing": {
|
|
"acc": true
|
|
},
|
|
"mmlu_medical_genetics": {
|
|
"acc": true
|
|
},
|
|
"mmlu_miscellaneous": {
|
|
"acc": true
|
|
},
|
|
"mmlu_moral_disputes": {
|
|
"acc": true
|
|
},
|
|
"mmlu_moral_scenarios": {
|
|
"acc": true
|
|
},
|
|
"mmlu_nutrition": {
|
|
"acc": true
|
|
},
|
|
"mmlu_other": {
|
|
"acc": true
|
|
},
|
|
"mmlu_philosophy": {
|
|
"acc": true
|
|
},
|
|
"mmlu_prehistory": {
|
|
"acc": true
|
|
},
|
|
"mmlu_professional_accounting": {
|
|
"acc": true
|
|
},
|
|
"mmlu_professional_law": {
|
|
"acc": true
|
|
},
|
|
"mmlu_professional_medicine": {
|
|
"acc": true
|
|
},
|
|
"mmlu_professional_psychology": {
|
|
"acc": true
|
|
},
|
|
"mmlu_public_relations": {
|
|
"acc": true
|
|
},
|
|
"mmlu_security_studies": {
|
|
"acc": true
|
|
},
|
|
"mmlu_social_sciences": {
|
|
"acc": true
|
|
},
|
|
"mmlu_sociology": {
|
|
"acc": true
|
|
},
|
|
"mmlu_stem": {
|
|
"acc": true
|
|
},
|
|
"mmlu_us_foreign_policy": {
|
|
"acc": true
|
|
},
|
|
"mmlu_virology": {
|
|
"acc": true
|
|
},
|
|
"mmlu_world_religions": {
|
|
"acc": true
|
|
},
|
|
"piqa": {
|
|
"acc": true,
|
|
"acc_norm": true
|
|
},
|
|
"truthfulqa_mc2": {
|
|
"acc": true
|
|
},
|
|
"winogrande": {
|
|
"acc": true
|
|
}
|
|
},
|
|
"n-samples": {
|
|
"winogrande": {
|
|
"original": 1267,
|
|
"effective": 1267
|
|
},
|
|
"truthfulqa_mc2": {
|
|
"original": 817,
|
|
"effective": 817
|
|
},
|
|
"piqa": {
|
|
"original": 1838,
|
|
"effective": 1838
|
|
},
|
|
"mmlu_abstract_algebra": {
|
|
"original": 100,
|
|
"effective": 100
|
|
},
|
|
"mmlu_anatomy": {
|
|
"original": 135,
|
|
"effective": 135
|
|
},
|
|
"mmlu_astronomy": {
|
|
"original": 152,
|
|
"effective": 152
|
|
},
|
|
"mmlu_college_biology": {
|
|
"original": 144,
|
|
"effective": 144
|
|
},
|
|
"mmlu_college_chemistry": {
|
|
"original": 100,
|
|
"effective": 100
|
|
},
|
|
"mmlu_college_computer_science": {
|
|
"original": 100,
|
|
"effective": 100
|
|
},
|
|
"mmlu_college_mathematics": {
|
|
"original": 100,
|
|
"effective": 100
|
|
},
|
|
"mmlu_college_physics": {
|
|
"original": 102,
|
|
"effective": 102
|
|
},
|
|
"mmlu_computer_security": {
|
|
"original": 100,
|
|
"effective": 100
|
|
},
|
|
"mmlu_conceptual_physics": {
|
|
"original": 235,
|
|
"effective": 235
|
|
},
|
|
"mmlu_electrical_engineering": {
|
|
"original": 145,
|
|
"effective": 145
|
|
},
|
|
"mmlu_elementary_mathematics": {
|
|
"original": 378,
|
|
"effective": 378
|
|
},
|
|
"mmlu_high_school_biology": {
|
|
"original": 310,
|
|
"effective": 310
|
|
},
|
|
"mmlu_high_school_chemistry": {
|
|
"original": 203,
|
|
"effective": 203
|
|
},
|
|
"mmlu_high_school_computer_science": {
|
|
"original": 100,
|
|
"effective": 100
|
|
},
|
|
"mmlu_high_school_mathematics": {
|
|
"original": 270,
|
|
"effective": 270
|
|
},
|
|
"mmlu_high_school_physics": {
|
|
"original": 151,
|
|
"effective": 151
|
|
},
|
|
"mmlu_high_school_statistics": {
|
|
"original": 216,
|
|
"effective": 216
|
|
},
|
|
"mmlu_machine_learning": {
|
|
"original": 112,
|
|
"effective": 112
|
|
},
|
|
"mmlu_business_ethics": {
|
|
"original": 100,
|
|
"effective": 100
|
|
},
|
|
"mmlu_clinical_knowledge": {
|
|
"original": 265,
|
|
"effective": 265
|
|
},
|
|
"mmlu_college_medicine": {
|
|
"original": 173,
|
|
"effective": 173
|
|
},
|
|
"mmlu_global_facts": {
|
|
"original": 100,
|
|
"effective": 100
|
|
},
|
|
"mmlu_human_aging": {
|
|
"original": 223,
|
|
"effective": 223
|
|
},
|
|
"mmlu_management": {
|
|
"original": 103,
|
|
"effective": 103
|
|
},
|
|
"mmlu_marketing": {
|
|
"original": 234,
|
|
"effective": 234
|
|
},
|
|
"mmlu_medical_genetics": {
|
|
"original": 100,
|
|
"effective": 100
|
|
},
|
|
"mmlu_miscellaneous": {
|
|
"original": 783,
|
|
"effective": 783
|
|
},
|
|
"mmlu_nutrition": {
|
|
"original": 306,
|
|
"effective": 306
|
|
},
|
|
"mmlu_professional_accounting": {
|
|
"original": 282,
|
|
"effective": 282
|
|
},
|
|
"mmlu_professional_medicine": {
|
|
"original": 272,
|
|
"effective": 272
|
|
},
|
|
"mmlu_virology": {
|
|
"original": 166,
|
|
"effective": 166
|
|
},
|
|
"mmlu_econometrics": {
|
|
"original": 114,
|
|
"effective": 114
|
|
},
|
|
"mmlu_high_school_geography": {
|
|
"original": 198,
|
|
"effective": 198
|
|
},
|
|
"mmlu_high_school_government_and_politics": {
|
|
"original": 193,
|
|
"effective": 193
|
|
},
|
|
"mmlu_high_school_macroeconomics": {
|
|
"original": 390,
|
|
"effective": 390
|
|
},
|
|
"mmlu_high_school_microeconomics": {
|
|
"original": 238,
|
|
"effective": 238
|
|
},
|
|
"mmlu_high_school_psychology": {
|
|
"original": 545,
|
|
"effective": 545
|
|
},
|
|
"mmlu_human_sexuality": {
|
|
"original": 131,
|
|
"effective": 131
|
|
},
|
|
"mmlu_professional_psychology": {
|
|
"original": 612,
|
|
"effective": 612
|
|
},
|
|
"mmlu_public_relations": {
|
|
"original": 110,
|
|
"effective": 110
|
|
},
|
|
"mmlu_security_studies": {
|
|
"original": 245,
|
|
"effective": 245
|
|
},
|
|
"mmlu_sociology": {
|
|
"original": 201,
|
|
"effective": 201
|
|
},
|
|
"mmlu_us_foreign_policy": {
|
|
"original": 100,
|
|
"effective": 100
|
|
},
|
|
"mmlu_formal_logic": {
|
|
"original": 126,
|
|
"effective": 126
|
|
},
|
|
"mmlu_high_school_european_history": {
|
|
"original": 165,
|
|
"effective": 165
|
|
},
|
|
"mmlu_high_school_us_history": {
|
|
"original": 204,
|
|
"effective": 204
|
|
},
|
|
"mmlu_high_school_world_history": {
|
|
"original": 237,
|
|
"effective": 237
|
|
},
|
|
"mmlu_international_law": {
|
|
"original": 121,
|
|
"effective": 121
|
|
},
|
|
"mmlu_jurisprudence": {
|
|
"original": 108,
|
|
"effective": 108
|
|
},
|
|
"mmlu_logical_fallacies": {
|
|
"original": 163,
|
|
"effective": 163
|
|
},
|
|
"mmlu_moral_disputes": {
|
|
"original": 346,
|
|
"effective": 346
|
|
},
|
|
"mmlu_moral_scenarios": {
|
|
"original": 895,
|
|
"effective": 895
|
|
},
|
|
"mmlu_philosophy": {
|
|
"original": 311,
|
|
"effective": 311
|
|
},
|
|
"mmlu_prehistory": {
|
|
"original": 324,
|
|
"effective": 324
|
|
},
|
|
"mmlu_professional_law": {
|
|
"original": 1534,
|
|
"effective": 1534
|
|
},
|
|
"mmlu_world_religions": {
|
|
"original": 171,
|
|
"effective": 171
|
|
},
|
|
"hellaswag": {
|
|
"original": 10042,
|
|
"effective": 10042
|
|
},
|
|
"gsm8k": {
|
|
"original": 1319,
|
|
"effective": 1319
|
|
},
|
|
"boolq": {
|
|
"original": 3270,
|
|
"effective": 3270
|
|
},
|
|
"arc_challenge": {
|
|
"original": 1172,
|
|
"effective": 1172
|
|
}
|
|
},
|
|
"config": {
|
|
"model": "hf",
|
|
"model_args": "pretrained=strykes/emberforge-3b-reasoner,trust_remote_code=True,dtype=float16",
|
|
"model_num_parameters": 3933637120,
|
|
"model_dtype": "torch.float16",
|
|
"model_revision": "main",
|
|
"model_sha": "8d89b5f0898a7c7ac824291f92609f7e79081035",
|
|
"batch_size": "auto",
|
|
"batch_sizes": [
|
|
8
|
|
],
|
|
"device": "cuda:0",
|
|
"use_cache": null,
|
|
"limit": null,
|
|
"bootstrap_iters": 100000,
|
|
"gen_kwargs": null,
|
|
"random_seed": 0,
|
|
"numpy_seed": 1234,
|
|
"torch_seed": 1234,
|
|
"fewshot_seed": 1234
|
|
},
|
|
"git_hash": null,
|
|
"date": 1771885256.465489,
|
|
"pretty_env_info": "PyTorch version: 2.4.1+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.4 LTS (x86_64)\nGCC version: Could not collect\nClang version: Could not collect\nCMake version: version 3.30.3\nLibc version: glibc-2.35\n\nPython version: 3.11.9 | packaged by conda-forge | (main, Apr 19 2024, 18:36:13) [GCC 12.3.0] (64-bit runtime)\nPython platform: Linux-5.15.0-161-generic-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: Could not collect\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: GPU 0: NVIDIA GeForce RTX 4090\nNvidia driver version: 580.95.05\ncuDNN version: Could not collect\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 43 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: AuthenticAMD\nModel name: AMD EPYC 7532 32-Core Processor\nCPU family: 23\nModel: 49\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 0\nFrequency boost: enabled\nCPU max MHz: 2400.0000\nCPU min MHz: 1500.0000\nBogoMIPS: 4800.21\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba ibrs ibpb stibp vmmcall fsgsbase bmi1 avx2 smep bmi2 cqm rdt_a rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local clzero irperf xsaveerptr rdpru wbnoinvd amd_ppin arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif v_spec_ctrl umip rdpid overflow_recov succor smca sme sev sev_es ibpb_exit_to_user\nVirtualization: AMD-V\nL1d cache: 2 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 32 MiB (64 instances)\nL3 cache: 512 MiB (32 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Gather data sampling: Not affected\nVulnerability Indirect target selection: Not affected\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Not affected\nVulnerability Reg file data sampling: Not affected\nVulnerability Retbleed: Mitigation; untrained return thunk; SMT enabled with STIBP protection\nVulnerability Spec rstack overflow: Mitigation; safe RET\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; IBPB conditional; STIBP always-on; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds: Not affected\nVulnerability Tsa: Not affected\nVulnerability Tsx async abort: Not affected\nVulnerability Vmscape: Mitigation; IBPB before exit to userspace\n\nVersions of relevant libraries:\n[pip3] numpy==2.1.1\n[pip3] optree==0.12.1\n[pip3] torch==2.4.1+cu121\n[pip3] torchaudio==2.4.1+cu121\n[pip3] torchelastic==0.2.2\n[pip3] torchvision==0.19.1+cu121\n[pip3] triton==3.0.0\n[conda] numpy 2.1.1 py311h71ddf71_0 conda-forge\n[conda] optree 0.12.1 pypi_0 pypi\n[conda] torch 2.4.1+cu121 pypi_0 pypi\n[conda] torchaudio 2.4.1+cu121 pypi_0 pypi\n[conda] torchelastic 0.2.2 pypi_0 pypi\n[conda] torchvision 0.19.1+cu121 pypi_0 pypi\n[conda] triton 3.0.0 pypi_0 pypi",
|
|
"transformers_version": "4.48.3",
|
|
"upper_git_hash": null,
|
|
"tokenizer_pad_token": [
|
|
"<unk>",
|
|
"0"
|
|
],
|
|
"tokenizer_eos_token": [
|
|
"<|im_end|>",
|
|
"166101"
|
|
],
|
|
"tokenizer_bos_token": [
|
|
"<|im_start|>",
|
|
"166100"
|
|
],
|
|
"eot_token_id": 166101,
|
|
"max_length": 262144,
|
|
"task_hashes": {},
|
|
"model_source": "hf",
|
|
"model_name": "strykes/emberforge-3b-reasoner",
|
|
"model_name_sanitized": "strykes__emberforge-3b-reasoner",
|
|
"system_instruction": null,
|
|
"system_instruction_sha": null,
|
|
"fewshot_as_multiturn": false,
|
|
"chat_template": null,
|
|
"chat_template_sha": null,
|
|
"start_time": 6705325.752072393,
|
|
"end_time": 6711657.301533823,
|
|
"total_evaluation_time_seconds": "6331.549461429939"
|
|
} |