4664 lines
154 KiB
JSON
4664 lines
154 KiB
JSON
|
{
|
||
|
|
"results": {
|
||
|
|
"winogrande": {
|
||
|
|
"name": "winogrande",
|
||
|
|
"alias": "winogrande",
|
||
|
|
"sample_len": 1267,
|
||
|
|
"acc,none": 0.49171270718232046,
|
||
|
|
"acc_stderr,none": 0.01405055532282417
|
||
|
|
},
|
||
|
|
"arc_easy": {
|
||
|
|
"name": "arc_easy",
|
||
|
|
"alias": "arc_easy",
|
||
|
|
"sample_len": 2376,
|
||
|
|
"acc,none": 0.4090909090909091,
|
||
|
|
"acc_stderr,none": 0.010088775152615642,
|
||
|
|
"acc_norm,none": 0.3859427609427609,
|
||
|
|
"acc_norm_stderr,none": 0.009989277329504026
|
||
|
|
},
|
||
|
|
"hellaswag": {
|
||
|
|
"name": "hellaswag",
|
||
|
|
"alias": "hellaswag",
|
||
|
|
"sample_len": 10042,
|
||
|
|
"acc,none": 0.2749452300338578,
|
||
|
|
"acc_stderr,none": 0.004455741817862,
|
||
|
|
"acc_norm,none": 0.28231428002389963,
|
||
|
|
"acc_norm_stderr,none": 0.004492055279407451
|
||
|
|
},
|
||
|
|
"boolq": {
|
||
|
|
"name": "boolq",
|
||
|
|
"alias": "boolq",
|
||
|
|
"sample_len": 3270,
|
||
|
|
"acc,none": 0.5850152905198777,
|
||
|
|
"acc_stderr,none": 0.008617716361921567
|
||
|
|
},
|
||
|
|
"arc_challenge": {
|
||
|
|
"name": "arc_challenge",
|
||
|
|
"alias": "arc_challenge",
|
||
|
|
"sample_len": 1172,
|
||
|
|
"acc,none": 0.19795221843003413,
|
||
|
|
"acc_stderr,none": 0.011643990971573369,
|
||
|
|
"acc_norm,none": 0.23122866894197952,
|
||
|
|
"acc_norm_stderr,none": 0.01232085883477234
|
||
|
|
},
|
||
|
|
"piqa": {
|
||
|
|
"name": "piqa",
|
||
|
|
"alias": "piqa",
|
||
|
|
"sample_len": 1838,
|
||
|
|
"acc,none": 0.5963003264417845,
|
||
|
|
"acc_stderr,none": 0.011447407541749126,
|
||
|
|
"acc_norm,none": 0.5761697497279652,
|
||
|
|
"acc_norm_stderr,none": 0.011529663270276148
|
||
|
|
},
|
||
|
|
"blimp_adjunct_island": {
|
||
|
|
"name": "blimp_adjunct_island",
|
||
|
|
"alias": "blimp_adjunct_island",
|
||
|
|
"sample_len": 1000,
|
||
|
|
"acc,none": 0.755,
|
||
|
|
"acc_stderr,none": 0.01360735683959821
|
||
|
|
},
|
||
|
|
"blimp_anaphor_gender_agreement": {
|
||
|
|
"name": "blimp_anaphor_gender_agreement",
|
||
|
|
"alias": "blimp_anaphor_gender_agreement",
|
||
|
|
"sample_len": 1000,
|
||
|
|
"acc,none": 0.844,
|
||
|
|
"acc_stderr,none": 0.011480235006122295
|
||
|
|
},
|
||
|
|
"blimp_anaphor_number_agreement": {
|
||
|
|
"name": "blimp_anaphor_number_agreement",
|
||
|
|
"alias": "blimp_anaphor_number_agreement",
|
||
|
|
"sample_len": 1000,
|
||
|
|
"acc,none": 0.909,
|
||
|
|
"acc_stderr,none": 0.009099549538400337
|
||
|
|
},
|
||
|
|
"blimp_animate_subject_passive": {
|
||
|
|
"name": "blimp_animate_subject_passive",
|
||
|
|
"alias": "blimp_animate_subject_passive",
|
||
|
|
"sample_len": 1000,
|
||
|
|
"acc,none": 0.719,
|
||
|
|
"acc_stderr,none": 0.01422115470843494
|
||
|
|
},
|
||
|
|
"blimp_animate_subject_trans": {
|
||
|
|
"name": "blimp_animate_subject_trans",
|
||
|
|
"alias": "blimp_animate_subject_trans",
|
||
|
|
"sample_len": 1000,
|
||
|
|
"acc,none": 0.726,
|
||
|
|
"acc_stderr,none": 0.01411109928825969
|
||
|
|
},
|
||
|
|
"blimp_causative": {
|
||
|
|
"name": "blimp_causative",
|
||
|
|
"alias": "blimp_causative",
|
||
|
|
"sample_len": 1000,
|
||
|
|
"acc,none": 0.533,
|
||
|
|
"acc_stderr,none": 0.015784807891138876
|
||
|
|
},
|
||
|
|
"blimp_complex_NP_island": {
|
||
|
|
"name": "blimp_complex_NP_island",
|
||
|
|
"alias": "blimp_complex_NP_island",
|
||
|
|
"sample_len": 1000,
|
||
|
|
"acc,none": 0.447,
|
||
|
|
"acc_stderr,none": 0.015730176046009098
|
||
|
|
},
|
||
|
|
"blimp_coordinate_structure_constraint_complex_left_branch": {
|
||
|
|
"name": "blimp_coordinate_structure_constraint_complex_left_branch",
|
||
|
|
"alias": "blimp_coordinate_structure_constraint_complex_left_branch",
|
||
|
|
"sample_len": 1000,
|
||
|
|
"acc,none": 0.672,
|
||
|
|
"acc_stderr,none": 0.01485384248727031
|
||
|
|
},
|
||
|
|
"blimp_coordinate_structure_constraint_object_extraction": {
|
||
|
|
"name": "blimp_coordinate_structure_constraint_object_extraction",
|
||
|
|
"alias": "blimp_coordinate_structure_constraint_object_extraction",
|
||
|
|
"sample_len": 1000,
|
||
|
|
"acc,none": 0.581,
|
||
|
|
"acc_stderr,none": 0.015610338967577953
|
||
|
|
},
|
||
|
|
"blimp_determiner_noun_agreement_1": {
|
||
|
|
"name": "blimp_determiner_noun_agreement_1",
|
||
|
|
"alias": "blimp_determiner_noun_agreement_1",
|
||
|
|
"sample_len": 1000,
|
||
|
|
"acc,none": 0.925,
|
||
|
|
"acc_stderr,none": 0.008333333333333333
|
||
|
|
},
|
||
|
|
"blimp_determiner_noun_agreement_2": {
|
||
|
|
"name": "blimp_determiner_noun_agreement_2",
|
||
|
|
"alias": "blimp_determiner_noun_agreement_2",
|
||
|
|
"sample_len": 1000,
|
||
|
|
"acc,none": 0.919,
|
||
|
|
"acc_stderr,none": 0.008632121032139986
|
||
|
|
},
|
||
|
|
"blimp_determiner_noun_agreement_irregular_1": {
|
||
|
|
"name": "blimp_determiner_noun_agreement_irregular_1",
|
||
|
|
"alias": "blimp_determiner_noun_agreement_irregular_1",
|
||
|
|
"sample_len": 1000,
|
||
|
|
"acc,none": 0.796,
|
||
|
|
"acc_stderr,none": 0.012749374359024311
|
||
|
|
},
|
||
|
|
"blimp_determiner_noun_agreement_irregular_2": {
|
||
|
|
"name": "blimp_determiner_noun_agreement_irregular_2",
|
||
|
|
"alias": "blimp_determiner_noun_agreement_irregular_2",
|
||
|
|
"sample_len": 1000,
|
||
|
|
"acc,none": 0.904,
|
||
|
|
"acc_stderr,none": 0.009320454434783257
|
||
|
|
},
|
||
|
|
"blimp_determiner_noun_agreement_with_adj_2": {
|
||
|
|
"name": "blimp_determiner_noun_agreement_with_adj_2",
|
||
|
|
"alias": "blimp_determiner_noun_agreement_with_adj_2",
|
||
|
|
"sample_len": 1000,
|
||
|
|
"acc,none": 0.888,
|
||
|
|
"acc_stderr,none": 0.009977753031397177
|
||
|
|
},
|
||
|
|
"blimp_determiner_noun_agreement_with_adj_irregular_1": {
|
||
|
|
"name": "blimp_determiner_noun_agreement_with_adj_irregular_1",
|
||
|
|
"alias": "blimp_determiner_noun_agreement_with_adj_irregular_1",
|
||
|
|
"sample_len": 1000,
|
||
|
|
"acc,none": 0.804,
|
||
|
|
"acc_stderr,none": 0.012559527926707347
|
||
|
|
},
|
||
|
|
"blimp_determiner_noun_agreement_with_adj_irregular_2": {
|
||
|
|
"name": "blimp_determiner_noun_agreement_with_adj_irregular_2",
|
||
|
|
"alias": "blimp_determiner_noun_agreement_with_adj_irregular_2",
|
||
|
|
"sample_len": 1000,
|
||
|
|
"acc,none": 0.827,
|
||
|
|
"acc_stderr,none": 0.011967214137559927
|
||
|
|
},
|
||
|
|
"blimp_determiner_noun_agreement_with_adjective_1": {
|
||
|
|
"name": "blimp_determiner_noun_agreement_with_adjective_1",
|
||
|
|
"alias": "blimp_determiner_noun_agreement_with_adjective_1",
|
||
|
|
"sample_len": 1000,
|
||
|
|
"acc,none": 0.892,
|
||
|
|
"acc_stderr,none": 0.009820001651345708
|
||
|
|
},
|
||
|
|
"blimp_distractor_agreement_relational_noun": {
|
||
|
|
"name": "blimp_distractor_agreement_relational_noun",
|
||
|
|
"alias": "blimp_distractor_agreement_relational_noun",
|
||
|
|
"sample_len": 1000,
|
||
|
|
"acc,none": 0.57,
|
||
|
|
"acc_stderr,none": 0.01566350361015521
|
||
|
|
},
|
||
|
|
"blimp_distractor_agreement_relative_clause": {
|
||
|
|
"name": "blimp_distractor_agreement_relative_clause",
|
||
|
|
"alias": "blimp_distractor_agreement_relative_clause",
|
||
|
|
"sample_len": 1000,
|
||
|
|
"acc,none": 0.477,
|
||
|
|
"acc_stderr,none": 0.015802554246726087
|
||
|
|
},
|
||
|
|
"blimp_drop_argument": {
|
||
|
|
"name": "blimp_drop_argument",
|
||
|
|
"alias": "blimp_drop_argument",
|
||
|
|
"sample_len": 1000,
|
||
|
|
"acc,none": 0.718,
|
||
|
|
"acc_stderr,none": 0.01423652621529138
|
||
|
|
},
|
||
|
|
"blimp_ellipsis_n_bar_1": {
|
||
|
|
"name": "blimp_ellipsis_n_bar_1",
|
||
|
|
"alias": "blimp_ellipsis_n_bar_1",
|
||
|
|
"sample_len": 1000,
|
||
|
|
"acc,none": 0.619,
|
||
|
|
"acc_stderr,none": 0.015364734787007391
|
||
|
|
},
|
||
|
|
"blimp_ellipsis_n_bar_2": {
|
||
|
|
"name": "blimp_ellipsis_n_bar_2",
|
||
|
|
"alias": "blimp_ellipsis_n_bar_2",
|
||
|
|
"sample_len": 1000,
|
||
|
|
"acc,none": 0.942,
|
||
|
|
"acc_stderr,none": 0.0073953154557929906
|
||
|
|
},
|
||
|
|
"blimp_existential_there_object_raising": {
|
||
|
|
"name": "blimp_existential_there_object_raising",
|
||
|
|
"alias": "blimp_existential_there_object_raising",
|
||
|
|
"sample_len": 1000,
|
||
|
|
"acc,none": 0.841,
|
||
|
|
"acc_stderr,none": 0.011569479368271195
|
||
|
|
},
|
||
|
|
"blimp_existential_there_quantifiers_1": {
|
||
|
|
"name": "blimp_existential_there_quantifiers_1",
|
||
|
|
"alias": "blimp_existential_there_quantifiers_1",
|
||
|
|
"sample_len": 1000,
|
||
|
|
"acc,none": 0.959,
|
||
|
|
"acc_stderr,none": 0.006273624021118764
|
||
|
|
},
|
||
|
|
"blimp_existential_there_quantifiers_2": {
|
||
|
|
"name": "blimp_existential_there_quantifiers_2",
|
||
|
|
"alias": "blimp_existential_there_quantifiers_2",
|
||
|
|
"sample_len": 1000,
|
||
|
|
"acc,none": 0.442,
|
||
|
|
"acc_stderr,none": 0.015712507211864152
|
||
|
|
},
|
||
|
|
"blimp_existential_there_subject_raising": {
|
||
|
|
"name": "blimp_existential_there_subject_raising",
|
||
|
|
"alias": "blimp_existential_there_subject_raising",
|
||
|
|
"sample_len": 1000,
|
||
|
|
"acc,none": 0.78,
|
||
|
|
"acc_stderr,none": 0.013106173040661862
|
||
|
|
},
|
||
|
|
"blimp_expletive_it_object_raising": {
|
||
|
|
"name": "blimp_expletive_it_object_raising",
|
||
|
|
"alias": "blimp_expletive_it_object_raising",
|
||
|
|
"sample_len": 1000,
|
||
|
|
"acc,none": 0.687,
|
||
|
|
"acc_stderr,none": 0.014671272822977775
|
||
|
|
},
|
||
|
|
"blimp_inchoative": {
|
||
|
|
"name": "blimp_inchoative",
|
||
|
|
"alias": "blimp_inchoative",
|
||
|
|
"sample_len": 1000,
|
||
|
|
"acc,none": 0.446,
|
||
|
|
"acc_stderr,none": 0.01572677116675039
|
||
|
|
},
|
||
|
|
"blimp_intransitive": {
|
||
|
|
"name": "blimp_intransitive",
|
||
|
|
"alias": "blimp_intransitive",
|
||
|
|
"sample_len": 1000,
|
||
|
|
"acc,none": 0.575,
|
||
|
|
"acc_stderr,none": 0.01564032031704017
|
||
|
|
},
|
||
|
|
"blimp_irregular_past_participle_adjectives": {
|
||
|
|
"name": "blimp_irregular_past_participle_adjectives",
|
||
|
|
"alias": "blimp_irregular_past_participle_adjectives",
|
||
|
|
"sample_len": 1000,
|
||
|
|
"acc,none": 0.801,
|
||
|
|
"acc_stderr,none": 0.01263164908309911
|
||
|
|
},
|
||
|
|
"blimp_irregular_past_participle_verbs": {
|
||
|
|
"name": "blimp_irregular_past_participle_verbs",
|
||
|
|
"alias": "blimp_irregular_past_participle_verbs",
|
||
|
|
"sample_len": 1000,
|
||
|
|
"acc,none": 0.694,
|
||
|
|
"acc_stderr,none": 0.014580006055436898
|
||
|
|
},
|
||
|
|
"blimp_irregular_plural_subject_verb_agreement_1": {
|
||
|
|
"name": "blimp_irregular_plural_subject_verb_agreement_1",
|
||
|
|
"alias": "blimp_irregular_plural_subject_verb_agreement_1",
|
||
|
|
"sample_len": 1000,
|
||
|
|
"acc,none": 0.781,
|
||
|
|
"acc_stderr,none": 0.013084731950262116
|
||
|
|
},
|
||
|
|
"blimp_irregular_plural_subject_verb_agreement_2": {
|
||
|
|
"name": "blimp_irregular_plural_subject_verb_agreement_2",
|
||
|
|
"alias": "blimp_irregular_plural_subject_verb_agreement_2",
|
||
|
|
"sample_len": 1000,
|
||
|
|
"acc,none": 0.819,
|
||
|
|
"acc_stderr,none": 0.012181436179178027
|
||
|
|
},
|
||
|
|
"blimp_left_branch_island_echo_question": {
|
||
|
|
"name": "blimp_left_branch_island_echo_question",
|
||
|
|
"alias": "blimp_left_branch_island_echo_question",
|
||
|
|
"sample_len": 1000,
|
||
|
|
"acc,none": 0.656,
|
||
|
|
"acc_stderr,none": 0.015029633724408919
|
||
|
|
},
|
||
|
|
"blimp_left_branch_island_simple_question": {
|
||
|
|
"name": "blimp_left_branch_island_simple_question",
|
||
|
|
"alias": "blimp_left_branch_island_simple_question",
|
||
|
|
"sample_len": 1000,
|
||
|
|
"acc,none": 0.625,
|
||
|
|
"acc_stderr,none": 0.015316971293620996
|
||
|
|
},
|
||
|
|
"blimp_matrix_question_npi_licensor_present": {
|
||
|
|
"name": "blimp_matrix_question_npi_licensor_present",
|
||
|
|
"alias": "blimp_matrix_question_npi_licensor_present",
|
||
|
|
"sample_len": 1000,
|
||
|
|
"acc,none": 0.074,
|
||
|
|
"acc_stderr,none": 0.0082820645127041
|
||
|
|
},
|
||
|
|
"blimp_npi_present_1": {
|
||
|
|
"name": "blimp_npi_present_1",
|
||
|
|
"alias": "blimp_npi_present_1",
|
||
|
|
"sample_len": 1000,
|
||
|
|
"acc,none": 0.341,
|
||
|
|
"acc_stderr,none": 0.01499813134840284
|
||
|
|
},
|
||
|
|
"blimp_npi_present_2": {
|
||
|
|
"name": "blimp_npi_present_2",
|
||
|
|
"alias": "blimp_npi_present_2",
|
||
|
|
"sample_len": 1000,
|
||
|
|
"acc,none": 0.262,
|
||
|
|
"acc_stderr,none": 0.013912208651021217
|
||
|
|
},
|
||
|
|
"blimp_only_npi_licensor_present": {
|
||
|
|
"name": "blimp_only_npi_licensor_present",
|
||
|
|
"alias": "blimp_only_npi_licensor_present",
|
||
|
|
"sample_len": 1000,
|
||
|
|
"acc,none": 0.875,
|
||
|
|
"acc_stderr,none": 0.010463483381956722
|
||
|
|
},
|
||
|
|
"blimp_only_npi_scope": {
|
||
|
|
"name": "blimp_only_npi_scope",
|
||
|
|
"alias": "blimp_only_npi_scope",
|
||
|
|
"sample_len": 1000,
|
||
|
|
"acc,none": 0.922,
|
||
|
|
"acc_stderr,none": 0.00848457353011853
|
||
|
|
},
|
||
|
|
"blimp_passive_1": {
|
||
|
|
"name": "blimp_passive_1",
|
||
|
|
"alias": "blimp_passive_1",
|
||
|
|
"sample_len": 1000,
|
||
|
|
"acc,none": 0.778,
|
||
|
|
"acc_stderr,none": 0.013148721948877349
|
||
|
|
},
|
||
|
|
"blimp_passive_2": {
|
||
|
|
"name": "blimp_passive_2",
|
||
|
|
"alias": "blimp_passive_2",
|
||
|
|
"sample_len": 1000,
|
||
|
|
"acc,none": 0.713,
|
||
|
|
"acc_stderr,none": 0.014312087053810001
|
||
|
|
},
|
||
|
|
"blimp_principle_A_c_command": {
|
||
|
|
"name": "blimp_principle_A_c_command",
|
||
|
|
"alias": "blimp_principle_A_c_command",
|
||
|
|
"sample_len": 1000,
|
||
|
|
"acc,none": 0.563,
|
||
|
|
"acc_stderr,none": 0.01569322392873043
|
||
|
|
},
|
||
|
|
"blimp_principle_A_case_1": {
|
||
|
|
"name": "blimp_principle_A_case_1",
|
||
|
|
"alias": "blimp_principle_A_case_1",
|
||
|
|
"sample_len": 1000,
|
||
|
|
"acc,none": 0.999,
|
||
|
|
"acc_stderr,none": 0.001
|
||
|
|
},
|
||
|
|
"blimp_principle_A_case_2": {
|
||
|
|
"name": "blimp_principle_A_case_2",
|
||
|
|
"alias": "blimp_principle_A_case_2",
|
||
|
|
"sample_len": 1000,
|
||
|
|
"acc,none": 0.801,
|
||
|
|
"acc_stderr,none": 0.01263164908309911
|
||
|
|
},
|
||
|
|
"blimp_principle_A_domain_1": {
|
||
|
|
"name": "blimp_principle_A_domain_1",
|
||
|
|
"alias": "blimp_principle_A_domain_1",
|
||
|
|
"sample_len": 1000,
|
||
|
|
"acc,none": 0.94,
|
||
|
|
"acc_stderr,none": 0.007513751157474971
|
||
|
|
},
|
||
|
|
"blimp_principle_A_domain_2": {
|
||
|
|
"name": "blimp_principle_A_domain_2",
|
||
|
|
"alias": "blimp_principle_A_domain_2",
|
||
|
|
"sample_len": 1000,
|
||
|
|
"acc,none": 0.713,
|
||
|
|
"acc_stderr,none": 0.014312087053810001
|
||
|
|
},
|
||
|
|
"blimp_principle_A_domain_3": {
|
||
|
|
"name": "blimp_principle_A_domain_3",
|
||
|
|
"alias": "blimp_principle_A_domain_3",
|
||
|
|
"sample_len": 1000,
|
||
|
|
"acc,none": 0.535,
|
||
|
|
"acc_stderr,none": 0.015780495050030086
|
||
|
|
},
|
||
|
|
"blimp_principle_A_reconstruction": {
|
||
|
|
"name": "blimp_principle_A_reconstruction",
|
||
|
|
"alias": "blimp_principle_A_reconstruction",
|
||
|
|
"sample_len": 1000,
|
||
|
|
"acc,none": 0.53,
|
||
|
|
"acc_stderr,none": 0.015790799515836725
|
||
|
|
},
|
||
|
|
"blimp_regular_plural_subject_verb_agreement_1": {
|
||
|
|
"name": "blimp_regular_plural_subject_verb_agreement_1",
|
||
|
|
"alias": "blimp_regular_plural_subject_verb_agreement_1",
|
||
|
|
"sample_len": 1000,
|
||
|
|
"acc,none": 0.751,
|
||
|
|
"acc_stderr,none": 0.013681600278702275
|
||
|
|
},
|
||
|
|
"blimp_regular_plural_subject_verb_agreement_2": {
|
||
|
|
"name": "blimp_regular_plural_subject_verb_agreement_2",
|
||
|
|
"alias": "blimp_regular_plural_subject_verb_agreement_2",
|
||
|
|
"sample_len": 1000,
|
||
|
|
"acc,none": 0.762,
|
||
|
|
"acc_stderr,none": 0.013473586661967093
|
||
|
|
},
|
||
|
|
"blimp_sentential_negation_npi_licensor_present": {
|
||
|
|
"name": "blimp_sentential_negation_npi_licensor_present",
|
||
|
|
"alias": "blimp_sentential_negation_npi_licensor_present",
|
||
|
|
"sample_len": 1000,
|
||
|
|
"acc,none": 0.995,
|
||
|
|
"acc_stderr,none": 0.0022315868748448895
|
||
|
|
},
|
||
|
|
"blimp_sentential_negation_npi_scope": {
|
||
|
|
"name": "blimp_sentential_negation_npi_scope",
|
||
|
|
"alias": "blimp_sentential_negation_npi_scope",
|
||
|
|
"sample_len": 1000,
|
||
|
|
"acc,none": 0.488,
|
||
|
|
"acc_stderr,none": 0.015814743314581686
|
||
|
|
},
|
||
|
|
"blimp_sentential_subject_island": {
|
||
|
|
"name": "blimp_sentential_subject_island",
|
||
|
|
"alias": "blimp_sentential_subject_island",
|
||
|
|
"sample_len": 1000,
|
||
|
|
"acc,none": 0.401,
|
||
|
|
"acc_stderr,none": 0.015506109745498368
|
||
|
|
},
|
||
|
|
"blimp_superlative_quantifiers_1": {
|
||
|
|
"name": "blimp_superlative_quantifiers_1",
|
||
|
|
"alias": "blimp_superlative_quantifiers_1",
|
||
|
|
"sample_len": 1000,
|
||
|
|
"acc,none": 0.738,
|
||
|
|
"acc_stderr,none": 0.013912208651021217
|
||
|
|
},
|
||
|
|
"blimp_superlative_quantifiers_2": {
|
||
|
|
"name": "blimp_superlative_quantifiers_2",
|
||
|
|
"alias": "blimp_superlative_quantifiers_2",
|
||
|
|
"sample_len": 1000,
|
||
|
|
"acc,none": 0.671,
|
||
|
|
"acc_stderr,none": 0.014865395385928355
|
||
|
|
},
|
||
|
|
"blimp_tough_vs_raising_1": {
|
||
|
|
"name": "blimp_tough_vs_raising_1",
|
||
|
|
"alias": "blimp_tough_vs_raising_1",
|
||
|
|
"sample_len": 1000,
|
||
|
|
"acc,none": 0.42,
|
||
|
|
"acc_stderr,none": 0.015615500115072968
|
||
|
|
},
|
||
|
|
"blimp_tough_vs_raising_2": {
|
||
|
|
"name": "blimp_tough_vs_raising_2",
|
||
|
|
"alias": "blimp_tough_vs_raising_2",
|
||
|
|
"sample_len": 1000,
|
||
|
|
"acc,none": 0.728,
|
||
|
|
"acc_stderr,none": 0.014078856992462642
|
||
|
|
},
|
||
|
|
"blimp_transitive": {
|
||
|
|
"name": "blimp_transitive",
|
||
|
|
"alias": "blimp_transitive",
|
||
|
|
"sample_len": 1000,
|
||
|
|
"acc,none": 0.722,
|
||
|
|
"acc_stderr,none": 0.014174516461485272
|
||
|
|
},
|
||
|
|
"blimp_wh_island": {
|
||
|
|
"name": "blimp_wh_island",
|
||
|
|
"alias": "blimp_wh_island",
|
||
|
|
"sample_len": 1000,
|
||
|
|
"acc,none": 0.602,
|
||
|
|
"acc_stderr,none": 0.015486634102859016
|
||
|
|
},
|
||
|
|
"blimp_wh_questions_object_gap": {
|
||
|
|
"name": "blimp_wh_questions_object_gap",
|
||
|
|
"alias": "blimp_wh_questions_object_gap",
|
||
|
|
"sample_len": 1000,
|
||
|
|
"acc,none": 0.639,
|
||
|
|
"acc_stderr,none": 0.015195720118175049
|
||
|
|
},
|
||
|
|
"blimp_wh_questions_subject_gap": {
|
||
|
|
"name": "blimp_wh_questions_subject_gap",
|
||
|
|
"alias": "blimp_wh_questions_subject_gap",
|
||
|
|
"sample_len": 1000,
|
||
|
|
"acc,none": 0.853,
|
||
|
|
"acc_stderr,none": 0.011203415395160376
|
||
|
|
},
|
||
|
|
"blimp_wh_questions_subject_gap_long_distance": {
|
||
|
|
"name": "blimp_wh_questions_subject_gap_long_distance",
|
||
|
|
"alias": "blimp_wh_questions_subject_gap_long_distance",
|
||
|
|
"sample_len": 1000,
|
||
|
|
"acc,none": 0.949,
|
||
|
|
"acc_stderr,none": 0.0069604200625714725
|
||
|
|
},
|
||
|
|
"blimp_wh_vs_that_no_gap": {
|
||
|
|
"name": "blimp_wh_vs_that_no_gap",
|
||
|
|
"alias": "blimp_wh_vs_that_no_gap",
|
||
|
|
"sample_len": 1000,
|
||
|
|
"acc,none": 0.879,
|
||
|
|
"acc_stderr,none": 0.010318210380946179
|
||
|
|
},
|
||
|
|
"blimp_wh_vs_that_no_gap_long_distance": {
|
||
|
|
"name": "blimp_wh_vs_that_no_gap_long_distance",
|
||
|
|
"alias": "blimp_wh_vs_that_no_gap_long_distance",
|
||
|
|
"sample_len": 1000,
|
||
|
|
"acc,none": 0.944,
|
||
|
|
"acc_stderr,none": 0.007274401481697097
|
||
|
|
},
|
||
|
|
"blimp_wh_vs_that_with_gap": {
|
||
|
|
"name": "blimp_wh_vs_that_with_gap",
|
||
|
|
"alias": "blimp_wh_vs_that_with_gap",
|
||
|
|
"sample_len": 1000,
|
||
|
|
"acc,none": 0.394,
|
||
|
|
"acc_stderr,none": 0.015459721957493382
|
||
|
|
},
|
||
|
|
"blimp_wh_vs_that_with_gap_long_distance": {
|
||
|
|
"name": "blimp_wh_vs_that_with_gap_long_distance",
|
||
|
|
"alias": "blimp_wh_vs_that_with_gap_long_distance",
|
||
|
|
"sample_len": 1000,
|
||
|
|
"acc,none": 0.178,
|
||
|
|
"acc_stderr,none": 0.012102167676183571
|
||
|
|
},
|
||
|
|
"wikitext": {
|
||
|
|
"name": "wikitext",
|
||
|
|
"alias": "wikitext",
|
||
|
|
"sample_len": 62,
|
||
|
|
"word_perplexity,none": 86.64177355483464,
|
||
|
|
"word_perplexity_stderr,none": "N/A",
|
||
|
|
"byte_perplexity,none": 2.3033752584676135,
|
||
|
|
"byte_perplexity_stderr,none": "N/A",
|
||
|
|
"bits_per_byte,none": 1.2037494695014286,
|
||
|
|
"bits_per_byte_stderr,none": "N/A"
|
||
|
|
},
|
||
|
|
"blimp": {
|
||
|
|
"alias": "blimp",
|
||
|
|
"name": "blimp",
|
||
|
|
"sample_len": 67000,
|
||
|
|
"acc,none": 0.6975074626865672,
|
||
|
|
"acc_stderr,none": 0.0015947299360720968,
|
||
|
|
"sample_count": {
|
||
|
|
"acc,none": 67000
|
||
|
|
}
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"groups": {
|
||
|
|
"blimp": {
|
||
|
|
"alias": "blimp",
|
||
|
|
"name": "blimp",
|
||
|
|
"sample_len": 67000,
|
||
|
|
"acc,none": 0.6975074626865672,
|
||
|
|
"acc_stderr,none": 0.0015947299360720968,
|
||
|
|
"sample_count": {
|
||
|
|
"acc,none": 67000
|
||
|
|
}
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"group_subtasks": {
|
||
|
|
"blimp": [
|
||
|
|
"blimp_adjunct_island",
|
||
|
|
"blimp_anaphor_gender_agreement",
|
||
|
|
"blimp_anaphor_number_agreement",
|
||
|
|
"blimp_animate_subject_passive",
|
||
|
|
"blimp_animate_subject_trans",
|
||
|
|
"blimp_causative",
|
||
|
|
"blimp_complex_NP_island",
|
||
|
|
"blimp_coordinate_structure_constraint_complex_left_branch",
|
||
|
|
"blimp_coordinate_structure_constraint_object_extraction",
|
||
|
|
"blimp_determiner_noun_agreement_1",
|
||
|
|
"blimp_determiner_noun_agreement_2",
|
||
|
|
"blimp_determiner_noun_agreement_irregular_1",
|
||
|
|
"blimp_determiner_noun_agreement_irregular_2",
|
||
|
|
"blimp_determiner_noun_agreement_with_adj_2",
|
||
|
|
"blimp_determiner_noun_agreement_with_adj_irregular_1",
|
||
|
|
"blimp_determiner_noun_agreement_with_adj_irregular_2",
|
||
|
|
"blimp_determiner_noun_agreement_with_adjective_1",
|
||
|
|
"blimp_distractor_agreement_relational_noun",
|
||
|
|
"blimp_distractor_agreement_relative_clause",
|
||
|
|
"blimp_drop_argument",
|
||
|
|
"blimp_ellipsis_n_bar_1",
|
||
|
|
"blimp_ellipsis_n_bar_2",
|
||
|
|
"blimp_existential_there_object_raising",
|
||
|
|
"blimp_existential_there_quantifiers_1",
|
||
|
|
"blimp_existential_there_quantifiers_2",
|
||
|
|
"blimp_existential_there_subject_raising",
|
||
|
|
"blimp_expletive_it_object_raising",
|
||
|
|
"blimp_inchoative",
|
||
|
|
"blimp_intransitive",
|
||
|
|
"blimp_irregular_past_participle_adjectives",
|
||
|
|
"blimp_irregular_past_participle_verbs",
|
||
|
|
"blimp_irregular_plural_subject_verb_agreement_1",
|
||
|
|
"blimp_irregular_plural_subject_verb_agreement_2",
|
||
|
|
"blimp_left_branch_island_echo_question",
|
||
|
|
"blimp_left_branch_island_simple_question",
|
||
|
|
"blimp_matrix_question_npi_licensor_present",
|
||
|
|
"blimp_npi_present_1",
|
||
|
|
"blimp_npi_present_2",
|
||
|
|
"blimp_only_npi_licensor_present",
|
||
|
|
"blimp_only_npi_scope",
|
||
|
|
"blimp_passive_1",
|
||
|
|
"blimp_passive_2",
|
||
|
|
"blimp_principle_A_c_command",
|
||
|
|
"blimp_principle_A_case_1",
|
||
|
|
"blimp_principle_A_case_2",
|
||
|
|
"blimp_principle_A_domain_1",
|
||
|
|
"blimp_principle_A_domain_2",
|
||
|
|
"blimp_principle_A_domain_3",
|
||
|
|
"blimp_principle_A_reconstruction",
|
||
|
|
"blimp_regular_plural_subject_verb_agreement_1",
|
||
|
|
"blimp_regular_plural_subject_verb_agreement_2",
|
||
|
|
"blimp_sentential_negation_npi_licensor_present",
|
||
|
|
"blimp_sentential_negation_npi_scope",
|
||
|
|
"blimp_sentential_subject_island",
|
||
|
|
"blimp_superlative_quantifiers_1",
|
||
|
|
"blimp_superlative_quantifiers_2",
|
||
|
|
"blimp_tough_vs_raising_1",
|
||
|
|
"blimp_tough_vs_raising_2",
|
||
|
|
"blimp_transitive",
|
||
|
|
"blimp_wh_island",
|
||
|
|
"blimp_wh_questions_object_gap",
|
||
|
|
"blimp_wh_questions_subject_gap",
|
||
|
|
"blimp_wh_questions_subject_gap_long_distance",
|
||
|
|
"blimp_wh_vs_that_no_gap",
|
||
|
|
"blimp_wh_vs_that_no_gap_long_distance",
|
||
|
|
"blimp_wh_vs_that_with_gap",
|
||
|
|
"blimp_wh_vs_that_with_gap_long_distance"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
"configs": {
|
||
|
|
"arc_challenge": {
|
||
|
|
"task": "arc_challenge",
|
||
|
|
"dataset_path": "allenai/ai2_arc",
|
||
|
|
"dataset_name": "ARC-Challenge",
|
||
|
|
"training_split": "train",
|
||
|
|
"validation_split": "validation",
|
||
|
|
"test_split": "test",
|
||
|
|
"doc_to_text": "Question: {{question}}\nAnswer:",
|
||
|
|
"doc_to_target": "{{choices.label.index(answerKey)}}",
|
||
|
|
"unsafe_code": false,
|
||
|
|
"doc_to_choice": "{{choices.text}}",
|
||
|
|
"description": "",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "default",
|
||
|
|
"split": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"samples": null,
|
||
|
|
"doc_to_text": "Question: {{question}}\nAnswer:",
|
||
|
|
"doc_to_choice": "{{choices.text}}",
|
||
|
|
"doc_to_target": "{{choices.label.index(answerKey)}}",
|
||
|
|
"gen_prefix": null,
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"metric": "acc_norm",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": true,
|
||
|
|
"doc_to_decontamination_query": "Question: {{question}}\nAnswer:",
|
||
|
|
"metadata": {
|
||
|
|
"version": 1.0,
|
||
|
|
"pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/",
|
||
|
|
"dtype": "float",
|
||
|
|
"config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/arc/arc_challenge.yaml"
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"arc_easy": {
|
||
|
|
"task": "arc_easy",
|
||
|
|
"dataset_path": "allenai/ai2_arc",
|
||
|
|
"dataset_name": "ARC-Easy",
|
||
|
|
"training_split": "train",
|
||
|
|
"validation_split": "validation",
|
||
|
|
"test_split": "test",
|
||
|
|
"doc_to_text": "Question: {{question}}\nAnswer:",
|
||
|
|
"doc_to_target": "{{choices.label.index(answerKey)}}",
|
||
|
|
"unsafe_code": false,
|
||
|
|
"doc_to_choice": "{{choices.text}}",
|
||
|
|
"description": "",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "default",
|
||
|
|
"split": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"samples": null,
|
||
|
|
"doc_to_text": "Question: {{question}}\nAnswer:",
|
||
|
|
"doc_to_choice": "{{choices.text}}",
|
||
|
|
"doc_to_target": "{{choices.label.index(answerKey)}}",
|
||
|
|
"gen_prefix": null,
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"metric": "acc_norm",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": true,
|
||
|
|
"doc_to_decontamination_query": "Question: {{question}}\nAnswer:",
|
||
|
|
"metadata": {
|
||
|
|
"version": 1.0,
|
||
|
|
"pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/",
|
||
|
|
"dtype": "float",
|
||
|
|
"config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/arc/arc_easy.yaml"
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"blimp_adjunct_island": {
|
||
|
|
"task": "blimp_adjunct_island",
|
||
|
|
"dataset_path": "nyu-mll/blimp",
|
||
|
|
"dataset_name": "adjunct_island",
|
||
|
|
"validation_split": "train",
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"unsafe_code": false,
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"description": "",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "default",
|
||
|
|
"split": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"samples": null,
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": true,
|
||
|
|
"doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
|
||
|
|
"metadata": {
|
||
|
|
"version": 1.0,
|
||
|
|
"pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/",
|
||
|
|
"dtype": "float",
|
||
|
|
"config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/adjunct_island.yaml"
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"blimp_anaphor_gender_agreement": {
|
||
|
|
"task": "blimp_anaphor_gender_agreement",
|
||
|
|
"dataset_path": "nyu-mll/blimp",
|
||
|
|
"dataset_name": "anaphor_gender_agreement",
|
||
|
|
"validation_split": "train",
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"unsafe_code": false,
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"description": "",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "default",
|
||
|
|
"split": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"samples": null,
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": true,
|
||
|
|
"doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
|
||
|
|
"metadata": {
|
||
|
|
"version": 1.0,
|
||
|
|
"pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/",
|
||
|
|
"dtype": "float",
|
||
|
|
"config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/anaphor_gender_agreement.yaml"
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"blimp_anaphor_number_agreement": {
|
||
|
|
"task": "blimp_anaphor_number_agreement",
|
||
|
|
"dataset_path": "nyu-mll/blimp",
|
||
|
|
"dataset_name": "anaphor_number_agreement",
|
||
|
|
"validation_split": "train",
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"unsafe_code": false,
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"description": "",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "default",
|
||
|
|
"split": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"samples": null,
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": true,
|
||
|
|
"doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
|
||
|
|
"metadata": {
|
||
|
|
"version": 1.0,
|
||
|
|
"pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/",
|
||
|
|
"dtype": "float",
|
||
|
|
"config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/anaphor_number_agreement.yaml"
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"blimp_animate_subject_passive": {
|
||
|
|
"task": "blimp_animate_subject_passive",
|
||
|
|
"dataset_path": "nyu-mll/blimp",
|
||
|
|
"dataset_name": "animate_subject_passive",
|
||
|
|
"validation_split": "train",
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"unsafe_code": false,
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"description": "",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "default",
|
||
|
|
"split": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"samples": null,
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": true,
|
||
|
|
"doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
|
||
|
|
"metadata": {
|
||
|
|
"version": 1.0,
|
||
|
|
"pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/",
|
||
|
|
"dtype": "float",
|
||
|
|
"config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/animate_subject_passive.yaml"
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"blimp_animate_subject_trans": {
|
||
|
|
"task": "blimp_animate_subject_trans",
|
||
|
|
"dataset_path": "nyu-mll/blimp",
|
||
|
|
"dataset_name": "animate_subject_trans",
|
||
|
|
"validation_split": "train",
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"unsafe_code": false,
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"description": "",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "default",
|
||
|
|
"split": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"samples": null,
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": true,
|
||
|
|
"doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
|
||
|
|
"metadata": {
|
||
|
|
"version": 1.0,
|
||
|
|
"pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/",
|
||
|
|
"dtype": "float",
|
||
|
|
"config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/animate_subject_trans.yaml"
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"blimp_causative": {
|
||
|
|
"task": "blimp_causative",
|
||
|
|
"dataset_path": "nyu-mll/blimp",
|
||
|
|
"dataset_name": "causative",
|
||
|
|
"validation_split": "train",
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"unsafe_code": false,
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"description": "",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "default",
|
||
|
|
"split": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"samples": null,
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": true,
|
||
|
|
"doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
|
||
|
|
"metadata": {
|
||
|
|
"version": 1.0,
|
||
|
|
"pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/",
|
||
|
|
"dtype": "float",
|
||
|
|
"config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/causative.yaml"
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"blimp_complex_NP_island": {
|
||
|
|
"task": "blimp_complex_NP_island",
|
||
|
|
"dataset_path": "nyu-mll/blimp",
|
||
|
|
"dataset_name": "complex_NP_island",
|
||
|
|
"validation_split": "train",
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"unsafe_code": false,
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"description": "",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "default",
|
||
|
|
"split": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"samples": null,
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": true,
|
||
|
|
"doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
|
||
|
|
"metadata": {
|
||
|
|
"version": 1.0,
|
||
|
|
"pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/",
|
||
|
|
"dtype": "float",
|
||
|
|
"config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/complex_NP_island.yaml"
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"blimp_coordinate_structure_constraint_complex_left_branch": {
|
||
|
|
"task": "blimp_coordinate_structure_constraint_complex_left_branch",
|
||
|
|
"dataset_path": "nyu-mll/blimp",
|
||
|
|
"dataset_name": "coordinate_structure_constraint_complex_left_branch",
|
||
|
|
"validation_split": "train",
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"unsafe_code": false,
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"description": "",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "default",
|
||
|
|
"split": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"samples": null,
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": true,
|
||
|
|
"doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
|
||
|
|
"metadata": {
|
||
|
|
"version": 1.0,
|
||
|
|
"pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/",
|
||
|
|
"dtype": "float",
|
||
|
|
"config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/coordinate_structure_constraint_complex_left_branch.yaml"
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"blimp_coordinate_structure_constraint_object_extraction": {
|
||
|
|
"task": "blimp_coordinate_structure_constraint_object_extraction",
|
||
|
|
"dataset_path": "nyu-mll/blimp",
|
||
|
|
"dataset_name": "coordinate_structure_constraint_object_extraction",
|
||
|
|
"validation_split": "train",
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"unsafe_code": false,
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"description": "",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "default",
|
||
|
|
"split": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"samples": null,
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": true,
|
||
|
|
"doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
|
||
|
|
"metadata": {
|
||
|
|
"version": 1.0,
|
||
|
|
"pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/",
|
||
|
|
"dtype": "float",
|
||
|
|
"config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/coordinate_structure_constraint_object_extraction.yaml"
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"blimp_determiner_noun_agreement_1": {
|
||
|
|
"task": "blimp_determiner_noun_agreement_1",
|
||
|
|
"dataset_path": "nyu-mll/blimp",
|
||
|
|
"dataset_name": "determiner_noun_agreement_1",
|
||
|
|
"validation_split": "train",
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"unsafe_code": false,
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"description": "",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "default",
|
||
|
|
"split": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"samples": null,
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": true,
|
||
|
|
"doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
|
||
|
|
"metadata": {
|
||
|
|
"version": 1.0,
|
||
|
|
"pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/",
|
||
|
|
"dtype": "float",
|
||
|
|
"config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/determiner_noun_agreement_1.yaml"
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"blimp_determiner_noun_agreement_2": {
|
||
|
|
"task": "blimp_determiner_noun_agreement_2",
|
||
|
|
"dataset_path": "nyu-mll/blimp",
|
||
|
|
"dataset_name": "determiner_noun_agreement_2",
|
||
|
|
"validation_split": "train",
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"unsafe_code": false,
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"description": "",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "default",
|
||
|
|
"split": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"samples": null,
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": true,
|
||
|
|
"doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
|
||
|
|
"metadata": {
|
||
|
|
"version": 1.0,
|
||
|
|
"pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/",
|
||
|
|
"dtype": "float",
|
||
|
|
"config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/determiner_noun_agreement_2.yaml"
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"blimp_determiner_noun_agreement_irregular_1": {
|
||
|
|
"task": "blimp_determiner_noun_agreement_irregular_1",
|
||
|
|
"dataset_path": "nyu-mll/blimp",
|
||
|
|
"dataset_name": "determiner_noun_agreement_irregular_1",
|
||
|
|
"validation_split": "train",
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"unsafe_code": false,
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"description": "",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "default",
|
||
|
|
"split": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"samples": null,
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": true,
|
||
|
|
"doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
|
||
|
|
"metadata": {
|
||
|
|
"version": 1.0,
|
||
|
|
"pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/",
|
||
|
|
"dtype": "float",
|
||
|
|
"config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_1.yaml"
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"blimp_determiner_noun_agreement_irregular_2": {
|
||
|
|
"task": "blimp_determiner_noun_agreement_irregular_2",
|
||
|
|
"dataset_path": "nyu-mll/blimp",
|
||
|
|
"dataset_name": "determiner_noun_agreement_irregular_2",
|
||
|
|
"validation_split": "train",
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"unsafe_code": false,
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"description": "",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "default",
|
||
|
|
"split": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"samples": null,
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": true,
|
||
|
|
"doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
|
||
|
|
"metadata": {
|
||
|
|
"version": 1.0,
|
||
|
|
"pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/",
|
||
|
|
"dtype": "float",
|
||
|
|
"config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_2.yaml"
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"blimp_determiner_noun_agreement_with_adj_2": {
|
||
|
|
"task": "blimp_determiner_noun_agreement_with_adj_2",
|
||
|
|
"dataset_path": "nyu-mll/blimp",
|
||
|
|
"dataset_name": "determiner_noun_agreement_with_adj_2",
|
||
|
|
"validation_split": "train",
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"unsafe_code": false,
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"description": "",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "default",
|
||
|
|
"split": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"samples": null,
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": true,
|
||
|
|
"doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
|
||
|
|
"metadata": {
|
||
|
|
"version": 1.0,
|
||
|
|
"pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/",
|
||
|
|
"dtype": "float",
|
||
|
|
"config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_2.yaml"
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"blimp_determiner_noun_agreement_with_adj_irregular_1": {
|
||
|
|
"task": "blimp_determiner_noun_agreement_with_adj_irregular_1",
|
||
|
|
"dataset_path": "nyu-mll/blimp",
|
||
|
|
"dataset_name": "determiner_noun_agreement_with_adj_irregular_1",
|
||
|
|
"validation_split": "train",
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"unsafe_code": false,
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"description": "",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "default",
|
||
|
|
"split": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"samples": null,
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": true,
|
||
|
|
"doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
|
||
|
|
"metadata": {
|
||
|
|
"version": 1.0,
|
||
|
|
"pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/",
|
||
|
|
"dtype": "float",
|
||
|
|
"config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_1.yaml"
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"blimp_determiner_noun_agreement_with_adj_irregular_2": {
|
||
|
|
"task": "blimp_determiner_noun_agreement_with_adj_irregular_2",
|
||
|
|
"dataset_path": "nyu-mll/blimp",
|
||
|
|
"dataset_name": "determiner_noun_agreement_with_adj_irregular_2",
|
||
|
|
"validation_split": "train",
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"unsafe_code": false,
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"description": "",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "default",
|
||
|
|
"split": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"samples": null,
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": true,
|
||
|
|
"doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
|
||
|
|
"metadata": {
|
||
|
|
"version": 1.0,
|
||
|
|
"pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/",
|
||
|
|
"dtype": "float",
|
||
|
|
"config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_2.yaml"
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"blimp_determiner_noun_agreement_with_adjective_1": {
|
||
|
|
"task": "blimp_determiner_noun_agreement_with_adjective_1",
|
||
|
|
"dataset_path": "nyu-mll/blimp",
|
||
|
|
"dataset_name": "determiner_noun_agreement_with_adjective_1",
|
||
|
|
"validation_split": "train",
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"unsafe_code": false,
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"description": "",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "default",
|
||
|
|
"split": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"samples": null,
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": true,
|
||
|
|
"doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
|
||
|
|
"metadata": {
|
||
|
|
"version": 1.0,
|
||
|
|
"pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/",
|
||
|
|
"dtype": "float",
|
||
|
|
"config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/determiner_noun_agreement_with_adjective_1.yaml"
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"blimp_distractor_agreement_relational_noun": {
|
||
|
|
"task": "blimp_distractor_agreement_relational_noun",
|
||
|
|
"dataset_path": "nyu-mll/blimp",
|
||
|
|
"dataset_name": "distractor_agreement_relational_noun",
|
||
|
|
"validation_split": "train",
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"unsafe_code": false,
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"description": "",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "default",
|
||
|
|
"split": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"samples": null,
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": true,
|
||
|
|
"doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
|
||
|
|
"metadata": {
|
||
|
|
"version": 1.0,
|
||
|
|
"pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/",
|
||
|
|
"dtype": "float",
|
||
|
|
"config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/distractor_agreement_relational_noun.yaml"
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"blimp_distractor_agreement_relative_clause": {
|
||
|
|
"task": "blimp_distractor_agreement_relative_clause",
|
||
|
|
"dataset_path": "nyu-mll/blimp",
|
||
|
|
"dataset_name": "distractor_agreement_relative_clause",
|
||
|
|
"validation_split": "train",
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"unsafe_code": false,
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"description": "",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "default",
|
||
|
|
"split": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"samples": null,
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": true,
|
||
|
|
"doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
|
||
|
|
"metadata": {
|
||
|
|
"version": 1.0,
|
||
|
|
"pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/",
|
||
|
|
"dtype": "float",
|
||
|
|
"config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/distractor_agreement_relative_clause.yaml"
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"blimp_drop_argument": {
|
||
|
|
"task": "blimp_drop_argument",
|
||
|
|
"dataset_path": "nyu-mll/blimp",
|
||
|
|
"dataset_name": "drop_argument",
|
||
|
|
"validation_split": "train",
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"unsafe_code": false,
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"description": "",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "default",
|
||
|
|
"split": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"samples": null,
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": true,
|
||
|
|
"doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
|
||
|
|
"metadata": {
|
||
|
|
"version": 1.0,
|
||
|
|
"pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/",
|
||
|
|
"dtype": "float",
|
||
|
|
"config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/drop_argument.yaml"
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"blimp_ellipsis_n_bar_1": {
|
||
|
|
"task": "blimp_ellipsis_n_bar_1",
|
||
|
|
"dataset_path": "nyu-mll/blimp",
|
||
|
|
"dataset_name": "ellipsis_n_bar_1",
|
||
|
|
"validation_split": "train",
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"unsafe_code": false,
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"description": "",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "default",
|
||
|
|
"split": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"samples": null,
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": true,
|
||
|
|
"doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
|
||
|
|
"metadata": {
|
||
|
|
"version": 1.0,
|
||
|
|
"pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/",
|
||
|
|
"dtype": "float",
|
||
|
|
"config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/ellipsis_n_bar_1.yaml"
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"blimp_ellipsis_n_bar_2": {
|
||
|
|
"task": "blimp_ellipsis_n_bar_2",
|
||
|
|
"dataset_path": "nyu-mll/blimp",
|
||
|
|
"dataset_name": "ellipsis_n_bar_2",
|
||
|
|
"validation_split": "train",
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"unsafe_code": false,
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"description": "",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "default",
|
||
|
|
"split": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"samples": null,
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": true,
|
||
|
|
"doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
|
||
|
|
"metadata": {
|
||
|
|
"version": 1.0,
|
||
|
|
"pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/",
|
||
|
|
"dtype": "float",
|
||
|
|
"config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/ellipsis_n_bar_2.yaml"
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"blimp_existential_there_object_raising": {
|
||
|
|
"task": "blimp_existential_there_object_raising",
|
||
|
|
"dataset_path": "nyu-mll/blimp",
|
||
|
|
"dataset_name": "existential_there_object_raising",
|
||
|
|
"validation_split": "train",
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"unsafe_code": false,
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"description": "",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "default",
|
||
|
|
"split": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"samples": null,
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": true,
|
||
|
|
"doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
|
||
|
|
"metadata": {
|
||
|
|
"version": 1.0,
|
||
|
|
"pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/",
|
||
|
|
"dtype": "float",
|
||
|
|
"config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/existential_there_object_raising.yaml"
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"blimp_existential_there_quantifiers_1": {
|
||
|
|
"task": "blimp_existential_there_quantifiers_1",
|
||
|
|
"dataset_path": "nyu-mll/blimp",
|
||
|
|
"dataset_name": "existential_there_quantifiers_1",
|
||
|
|
"validation_split": "train",
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"unsafe_code": false,
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"description": "",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "default",
|
||
|
|
"split": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"samples": null,
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": true,
|
||
|
|
"doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
|
||
|
|
"metadata": {
|
||
|
|
"version": 1.0,
|
||
|
|
"pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/",
|
||
|
|
"dtype": "float",
|
||
|
|
"config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/existential_there_quantifiers_1.yaml"
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"blimp_existential_there_quantifiers_2": {
|
||
|
|
"task": "blimp_existential_there_quantifiers_2",
|
||
|
|
"dataset_path": "nyu-mll/blimp",
|
||
|
|
"dataset_name": "existential_there_quantifiers_2",
|
||
|
|
"validation_split": "train",
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"unsafe_code": false,
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"description": "",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "default",
|
||
|
|
"split": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"samples": null,
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": true,
|
||
|
|
"doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
|
||
|
|
"metadata": {
|
||
|
|
"version": 1.0,
|
||
|
|
"pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/",
|
||
|
|
"dtype": "float",
|
||
|
|
"config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/existential_there_quantifiers_2.yaml"
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"blimp_existential_there_subject_raising": {
|
||
|
|
"task": "blimp_existential_there_subject_raising",
|
||
|
|
"dataset_path": "nyu-mll/blimp",
|
||
|
|
"dataset_name": "existential_there_subject_raising",
|
||
|
|
"validation_split": "train",
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"unsafe_code": false,
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"description": "",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "default",
|
||
|
|
"split": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"samples": null,
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": true,
|
||
|
|
"doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
|
||
|
|
"metadata": {
|
||
|
|
"version": 1.0,
|
||
|
|
"pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/",
|
||
|
|
"dtype": "float",
|
||
|
|
"config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/existential_there_subject_raising.yaml"
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"blimp_expletive_it_object_raising": {
|
||
|
|
"task": "blimp_expletive_it_object_raising",
|
||
|
|
"dataset_path": "nyu-mll/blimp",
|
||
|
|
"dataset_name": "expletive_it_object_raising",
|
||
|
|
"validation_split": "train",
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"unsafe_code": false,
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"description": "",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "default",
|
||
|
|
"split": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"samples": null,
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": true,
|
||
|
|
"doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
|
||
|
|
"metadata": {
|
||
|
|
"version": 1.0,
|
||
|
|
"pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/",
|
||
|
|
"dtype": "float",
|
||
|
|
"config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/expletive_it_object_raising.yaml"
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"blimp_inchoative": {
|
||
|
|
"task": "blimp_inchoative",
|
||
|
|
"dataset_path": "nyu-mll/blimp",
|
||
|
|
"dataset_name": "inchoative",
|
||
|
|
"validation_split": "train",
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"unsafe_code": false,
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"description": "",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "default",
|
||
|
|
"split": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"samples": null,
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": true,
|
||
|
|
"doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
|
||
|
|
"metadata": {
|
||
|
|
"version": 1.0,
|
||
|
|
"pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/",
|
||
|
|
"dtype": "float",
|
||
|
|
"config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/inchoative.yaml"
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"blimp_intransitive": {
|
||
|
|
"task": "blimp_intransitive",
|
||
|
|
"dataset_path": "nyu-mll/blimp",
|
||
|
|
"dataset_name": "intransitive",
|
||
|
|
"validation_split": "train",
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"unsafe_code": false,
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"description": "",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "default",
|
||
|
|
"split": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"samples": null,
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": true,
|
||
|
|
"doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
|
||
|
|
"metadata": {
|
||
|
|
"version": 1.0,
|
||
|
|
"pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/",
|
||
|
|
"dtype": "float",
|
||
|
|
"config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/intransitive.yaml"
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"blimp_irregular_past_participle_adjectives": {
|
||
|
|
"task": "blimp_irregular_past_participle_adjectives",
|
||
|
|
"dataset_path": "nyu-mll/blimp",
|
||
|
|
"dataset_name": "irregular_past_participle_adjectives",
|
||
|
|
"validation_split": "train",
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"unsafe_code": false,
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"description": "",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "default",
|
||
|
|
"split": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"samples": null,
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": true,
|
||
|
|
"doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
|
||
|
|
"metadata": {
|
||
|
|
"version": 1.0,
|
||
|
|
"pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/",
|
||
|
|
"dtype": "float",
|
||
|
|
"config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/irregular_past_participle_adjectives.yaml"
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"blimp_irregular_past_participle_verbs": {
|
||
|
|
"task": "blimp_irregular_past_participle_verbs",
|
||
|
|
"dataset_path": "nyu-mll/blimp",
|
||
|
|
"dataset_name": "irregular_past_participle_verbs",
|
||
|
|
"validation_split": "train",
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"unsafe_code": false,
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"description": "",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "default",
|
||
|
|
"split": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"samples": null,
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": true,
|
||
|
|
"doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
|
||
|
|
"metadata": {
|
||
|
|
"version": 1.0,
|
||
|
|
"pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/",
|
||
|
|
"dtype": "float",
|
||
|
|
"config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/irregular_past_participle_verbs.yaml"
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"blimp_irregular_plural_subject_verb_agreement_1": {
|
||
|
|
"task": "blimp_irregular_plural_subject_verb_agreement_1",
|
||
|
|
"dataset_path": "nyu-mll/blimp",
|
||
|
|
"dataset_name": "irregular_plural_subject_verb_agreement_1",
|
||
|
|
"validation_split": "train",
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"unsafe_code": false,
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"description": "",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "default",
|
||
|
|
"split": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"samples": null,
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": true,
|
||
|
|
"doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
|
||
|
|
"metadata": {
|
||
|
|
"version": 1.0,
|
||
|
|
"pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/",
|
||
|
|
"dtype": "float",
|
||
|
|
"config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_1.yaml"
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"blimp_irregular_plural_subject_verb_agreement_2": {
|
||
|
|
"task": "blimp_irregular_plural_subject_verb_agreement_2",
|
||
|
|
"dataset_path": "nyu-mll/blimp",
|
||
|
|
"dataset_name": "irregular_plural_subject_verb_agreement_2",
|
||
|
|
"validation_split": "train",
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"unsafe_code": false,
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"description": "",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "default",
|
||
|
|
"split": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"samples": null,
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": true,
|
||
|
|
"doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
|
||
|
|
"metadata": {
|
||
|
|
"version": 1.0,
|
||
|
|
"pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/",
|
||
|
|
"dtype": "float",
|
||
|
|
"config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_2.yaml"
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"blimp_left_branch_island_echo_question": {
|
||
|
|
"task": "blimp_left_branch_island_echo_question",
|
||
|
|
"dataset_path": "nyu-mll/blimp",
|
||
|
|
"dataset_name": "left_branch_island_echo_question",
|
||
|
|
"validation_split": "train",
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"unsafe_code": false,
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"description": "",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "default",
|
||
|
|
"split": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"samples": null,
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": true,
|
||
|
|
"doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
|
||
|
|
"metadata": {
|
||
|
|
"version": 1.0,
|
||
|
|
"pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/",
|
||
|
|
"dtype": "float",
|
||
|
|
"config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/left_branch_island_echo_question.yaml"
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"blimp_left_branch_island_simple_question": {
|
||
|
|
"task": "blimp_left_branch_island_simple_question",
|
||
|
|
"dataset_path": "nyu-mll/blimp",
|
||
|
|
"dataset_name": "left_branch_island_simple_question",
|
||
|
|
"validation_split": "train",
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"unsafe_code": false,
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"description": "",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "default",
|
||
|
|
"split": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"samples": null,
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": true,
|
||
|
|
"doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
|
||
|
|
"metadata": {
|
||
|
|
"version": 1.0,
|
||
|
|
"pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/",
|
||
|
|
"dtype": "float",
|
||
|
|
"config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/left_branch_island_simple_question.yaml"
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"blimp_matrix_question_npi_licensor_present": {
|
||
|
|
"task": "blimp_matrix_question_npi_licensor_present",
|
||
|
|
"dataset_path": "nyu-mll/blimp",
|
||
|
|
"dataset_name": "matrix_question_npi_licensor_present",
|
||
|
|
"validation_split": "train",
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"unsafe_code": false,
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"description": "",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "default",
|
||
|
|
"split": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"samples": null,
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": true,
|
||
|
|
"doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
|
||
|
|
"metadata": {
|
||
|
|
"version": 1.0,
|
||
|
|
"pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/",
|
||
|
|
"dtype": "float",
|
||
|
|
"config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/matrix_question_npi_licensor_present.yaml"
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"blimp_npi_present_1": {
|
||
|
|
"task": "blimp_npi_present_1",
|
||
|
|
"dataset_path": "nyu-mll/blimp",
|
||
|
|
"dataset_name": "npi_present_1",
|
||
|
|
"validation_split": "train",
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"unsafe_code": false,
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"description": "",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "default",
|
||
|
|
"split": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"samples": null,
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": true,
|
||
|
|
"doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
|
||
|
|
"metadata": {
|
||
|
|
"version": 1.0,
|
||
|
|
"pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/",
|
||
|
|
"dtype": "float",
|
||
|
|
"config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/npi_present_1.yaml"
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"blimp_npi_present_2": {
|
||
|
|
"task": "blimp_npi_present_2",
|
||
|
|
"dataset_path": "nyu-mll/blimp",
|
||
|
|
"dataset_name": "npi_present_2",
|
||
|
|
"validation_split": "train",
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"unsafe_code": false,
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"description": "",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "default",
|
||
|
|
"split": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"samples": null,
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": true,
|
||
|
|
"doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
|
||
|
|
"metadata": {
|
||
|
|
"version": 1.0,
|
||
|
|
"pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/",
|
||
|
|
"dtype": "float",
|
||
|
|
"config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/npi_present_2.yaml"
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"blimp_only_npi_licensor_present": {
|
||
|
|
"task": "blimp_only_npi_licensor_present",
|
||
|
|
"dataset_path": "nyu-mll/blimp",
|
||
|
|
"dataset_name": "only_npi_licensor_present",
|
||
|
|
"validation_split": "train",
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"unsafe_code": false,
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"description": "",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "default",
|
||
|
|
"split": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"samples": null,
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": true,
|
||
|
|
"doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
|
||
|
|
"metadata": {
|
||
|
|
"version": 1.0,
|
||
|
|
"pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/",
|
||
|
|
"dtype": "float",
|
||
|
|
"config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/only_npi_licensor_present.yaml"
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"blimp_only_npi_scope": {
|
||
|
|
"task": "blimp_only_npi_scope",
|
||
|
|
"dataset_path": "nyu-mll/blimp",
|
||
|
|
"dataset_name": "only_npi_scope",
|
||
|
|
"validation_split": "train",
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"unsafe_code": false,
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"description": "",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "default",
|
||
|
|
"split": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"samples": null,
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": true,
|
||
|
|
"doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
|
||
|
|
"metadata": {
|
||
|
|
"version": 1.0,
|
||
|
|
"pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/",
|
||
|
|
"dtype": "float",
|
||
|
|
"config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/only_npi_scope.yaml"
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"blimp_passive_1": {
|
||
|
|
"task": "blimp_passive_1",
|
||
|
|
"dataset_path": "nyu-mll/blimp",
|
||
|
|
"dataset_name": "passive_1",
|
||
|
|
"validation_split": "train",
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"unsafe_code": false,
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"description": "",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "default",
|
||
|
|
"split": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"samples": null,
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": true,
|
||
|
|
"doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
|
||
|
|
"metadata": {
|
||
|
|
"version": 1.0,
|
||
|
|
"pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/",
|
||
|
|
"dtype": "float",
|
||
|
|
"config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/passive_1.yaml"
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"blimp_passive_2": {
|
||
|
|
"task": "blimp_passive_2",
|
||
|
|
"dataset_path": "nyu-mll/blimp",
|
||
|
|
"dataset_name": "passive_2",
|
||
|
|
"validation_split": "train",
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"unsafe_code": false,
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"description": "",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "default",
|
||
|
|
"split": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"samples": null,
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": true,
|
||
|
|
"doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
|
||
|
|
"metadata": {
|
||
|
|
"version": 1.0,
|
||
|
|
"pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/",
|
||
|
|
"dtype": "float",
|
||
|
|
"config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/passive_2.yaml"
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"blimp_principle_A_c_command": {
|
||
|
|
"task": "blimp_principle_A_c_command",
|
||
|
|
"dataset_path": "nyu-mll/blimp",
|
||
|
|
"dataset_name": "principle_A_c_command",
|
||
|
|
"validation_split": "train",
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"unsafe_code": false,
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"description": "",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "default",
|
||
|
|
"split": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"samples": null,
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": true,
|
||
|
|
"doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
|
||
|
|
"metadata": {
|
||
|
|
"version": 1.0,
|
||
|
|
"pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/",
|
||
|
|
"dtype": "float",
|
||
|
|
"config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/principle_A_c_command.yaml"
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"blimp_principle_A_case_1": {
|
||
|
|
"task": "blimp_principle_A_case_1",
|
||
|
|
"dataset_path": "nyu-mll/blimp",
|
||
|
|
"dataset_name": "principle_A_case_1",
|
||
|
|
"validation_split": "train",
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"unsafe_code": false,
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"description": "",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "default",
|
||
|
|
"split": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"samples": null,
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": true,
|
||
|
|
"doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
|
||
|
|
"metadata": {
|
||
|
|
"version": 1.0,
|
||
|
|
"pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/",
|
||
|
|
"dtype": "float",
|
||
|
|
"config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/principle_A_case_1.yaml"
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"blimp_principle_A_case_2": {
|
||
|
|
"task": "blimp_principle_A_case_2",
|
||
|
|
"dataset_path": "nyu-mll/blimp",
|
||
|
|
"dataset_name": "principle_A_case_2",
|
||
|
|
"validation_split": "train",
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"unsafe_code": false,
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"description": "",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "default",
|
||
|
|
"split": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"samples": null,
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": true,
|
||
|
|
"doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
|
||
|
|
"metadata": {
|
||
|
|
"version": 1.0,
|
||
|
|
"pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/",
|
||
|
|
"dtype": "float",
|
||
|
|
"config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/principle_A_case_2.yaml"
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"blimp_principle_A_domain_1": {
|
||
|
|
"task": "blimp_principle_A_domain_1",
|
||
|
|
"dataset_path": "nyu-mll/blimp",
|
||
|
|
"dataset_name": "principle_A_domain_1",
|
||
|
|
"validation_split": "train",
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"unsafe_code": false,
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"description": "",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "default",
|
||
|
|
"split": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"samples": null,
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": true,
|
||
|
|
"doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
|
||
|
|
"metadata": {
|
||
|
|
"version": 1.0,
|
||
|
|
"pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/",
|
||
|
|
"dtype": "float",
|
||
|
|
"config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/principle_A_domain_1.yaml"
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"blimp_principle_A_domain_2": {
|
||
|
|
"task": "blimp_principle_A_domain_2",
|
||
|
|
"dataset_path": "nyu-mll/blimp",
|
||
|
|
"dataset_name": "principle_A_domain_2",
|
||
|
|
"validation_split": "train",
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"unsafe_code": false,
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"description": "",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "default",
|
||
|
|
"split": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"samples": null,
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": true,
|
||
|
|
"doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
|
||
|
|
"metadata": {
|
||
|
|
"version": 1.0,
|
||
|
|
"pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/",
|
||
|
|
"dtype": "float",
|
||
|
|
"config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/principle_A_domain_2.yaml"
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"blimp_principle_A_domain_3": {
|
||
|
|
"task": "blimp_principle_A_domain_3",
|
||
|
|
"dataset_path": "nyu-mll/blimp",
|
||
|
|
"dataset_name": "principle_A_domain_3",
|
||
|
|
"validation_split": "train",
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"unsafe_code": false,
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"description": "",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "default",
|
||
|
|
"split": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"samples": null,
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": true,
|
||
|
|
"doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
|
||
|
|
"metadata": {
|
||
|
|
"version": 1.0,
|
||
|
|
"pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/",
|
||
|
|
"dtype": "float",
|
||
|
|
"config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/principle_A_domain_3.yaml"
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"blimp_principle_A_reconstruction": {
|
||
|
|
"task": "blimp_principle_A_reconstruction",
|
||
|
|
"dataset_path": "nyu-mll/blimp",
|
||
|
|
"dataset_name": "principle_A_reconstruction",
|
||
|
|
"validation_split": "train",
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"unsafe_code": false,
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"description": "",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "default",
|
||
|
|
"split": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"samples": null,
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": true,
|
||
|
|
"doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
|
||
|
|
"metadata": {
|
||
|
|
"version": 1.0,
|
||
|
|
"pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/",
|
||
|
|
"dtype": "float",
|
||
|
|
"config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/principle_A_reconstruction.yaml"
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"blimp_regular_plural_subject_verb_agreement_1": {
|
||
|
|
"task": "blimp_regular_plural_subject_verb_agreement_1",
|
||
|
|
"dataset_path": "nyu-mll/blimp",
|
||
|
|
"dataset_name": "regular_plural_subject_verb_agreement_1",
|
||
|
|
"validation_split": "train",
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"unsafe_code": false,
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"description": "",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "default",
|
||
|
|
"split": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"samples": null,
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": true,
|
||
|
|
"doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
|
||
|
|
"metadata": {
|
||
|
|
"version": 1.0,
|
||
|
|
"pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/",
|
||
|
|
"dtype": "float",
|
||
|
|
"config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_1.yaml"
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"blimp_regular_plural_subject_verb_agreement_2": {
|
||
|
|
"task": "blimp_regular_plural_subject_verb_agreement_2",
|
||
|
|
"dataset_path": "nyu-mll/blimp",
|
||
|
|
"dataset_name": "regular_plural_subject_verb_agreement_2",
|
||
|
|
"validation_split": "train",
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"unsafe_code": false,
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"description": "",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "default",
|
||
|
|
"split": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"samples": null,
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": true,
|
||
|
|
"doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
|
||
|
|
"metadata": {
|
||
|
|
"version": 1.0,
|
||
|
|
"pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/",
|
||
|
|
"dtype": "float",
|
||
|
|
"config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_2.yaml"
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"blimp_sentential_negation_npi_licensor_present": {
|
||
|
|
"task": "blimp_sentential_negation_npi_licensor_present",
|
||
|
|
"dataset_path": "nyu-mll/blimp",
|
||
|
|
"dataset_name": "sentential_negation_npi_licensor_present",
|
||
|
|
"validation_split": "train",
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"unsafe_code": false,
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"description": "",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "default",
|
||
|
|
"split": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"samples": null,
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": true,
|
||
|
|
"doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
|
||
|
|
"metadata": {
|
||
|
|
"version": 1.0,
|
||
|
|
"pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/",
|
||
|
|
"dtype": "float",
|
||
|
|
"config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/sentential_negation_npi_licensor_present.yaml"
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"blimp_sentential_negation_npi_scope": {
|
||
|
|
"task": "blimp_sentential_negation_npi_scope",
|
||
|
|
"dataset_path": "nyu-mll/blimp",
|
||
|
|
"dataset_name": "sentential_negation_npi_scope",
|
||
|
|
"validation_split": "train",
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"unsafe_code": false,
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"description": "",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "default",
|
||
|
|
"split": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"samples": null,
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": true,
|
||
|
|
"doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
|
||
|
|
"metadata": {
|
||
|
|
"version": 1.0,
|
||
|
|
"pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/",
|
||
|
|
"dtype": "float",
|
||
|
|
"config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/sentential_negation_npi_scope.yaml"
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"blimp_sentential_subject_island": {
|
||
|
|
"task": "blimp_sentential_subject_island",
|
||
|
|
"dataset_path": "nyu-mll/blimp",
|
||
|
|
"dataset_name": "sentential_subject_island",
|
||
|
|
"validation_split": "train",
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"unsafe_code": false,
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"description": "",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "default",
|
||
|
|
"split": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"samples": null,
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": true,
|
||
|
|
"doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
|
||
|
|
"metadata": {
|
||
|
|
"version": 1.0,
|
||
|
|
"pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/",
|
||
|
|
"dtype": "float",
|
||
|
|
"config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/sentential_subject_island.yaml"
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"blimp_superlative_quantifiers_1": {
|
||
|
|
"task": "blimp_superlative_quantifiers_1",
|
||
|
|
"dataset_path": "nyu-mll/blimp",
|
||
|
|
"dataset_name": "superlative_quantifiers_1",
|
||
|
|
"validation_split": "train",
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"unsafe_code": false,
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"description": "",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "default",
|
||
|
|
"split": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"samples": null,
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": true,
|
||
|
|
"doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
|
||
|
|
"metadata": {
|
||
|
|
"version": 1.0,
|
||
|
|
"pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/",
|
||
|
|
"dtype": "float",
|
||
|
|
"config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/superlative_quantifiers_1.yaml"
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"blimp_superlative_quantifiers_2": {
|
||
|
|
"task": "blimp_superlative_quantifiers_2",
|
||
|
|
"dataset_path": "nyu-mll/blimp",
|
||
|
|
"dataset_name": "superlative_quantifiers_2",
|
||
|
|
"validation_split": "train",
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"unsafe_code": false,
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"description": "",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "default",
|
||
|
|
"split": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"samples": null,
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": true,
|
||
|
|
"doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
|
||
|
|
"metadata": {
|
||
|
|
"version": 1.0,
|
||
|
|
"pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/",
|
||
|
|
"dtype": "float",
|
||
|
|
"config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/superlative_quantifiers_2.yaml"
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"blimp_tough_vs_raising_1": {
|
||
|
|
"task": "blimp_tough_vs_raising_1",
|
||
|
|
"dataset_path": "nyu-mll/blimp",
|
||
|
|
"dataset_name": "tough_vs_raising_1",
|
||
|
|
"validation_split": "train",
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"unsafe_code": false,
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"description": "",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "default",
|
||
|
|
"split": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"samples": null,
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": true,
|
||
|
|
"doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
|
||
|
|
"metadata": {
|
||
|
|
"version": 1.0,
|
||
|
|
"pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/",
|
||
|
|
"dtype": "float",
|
||
|
|
"config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/tough_vs_raising_1.yaml"
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"blimp_tough_vs_raising_2": {
|
||
|
|
"task": "blimp_tough_vs_raising_2",
|
||
|
|
"dataset_path": "nyu-mll/blimp",
|
||
|
|
"dataset_name": "tough_vs_raising_2",
|
||
|
|
"validation_split": "train",
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"unsafe_code": false,
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"description": "",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "default",
|
||
|
|
"split": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"samples": null,
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": true,
|
||
|
|
"doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
|
||
|
|
"metadata": {
|
||
|
|
"version": 1.0,
|
||
|
|
"pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/",
|
||
|
|
"dtype": "float",
|
||
|
|
"config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/tough_vs_raising_2.yaml"
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"blimp_transitive": {
|
||
|
|
"task": "blimp_transitive",
|
||
|
|
"dataset_path": "nyu-mll/blimp",
|
||
|
|
"dataset_name": "transitive",
|
||
|
|
"validation_split": "train",
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"unsafe_code": false,
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"description": "",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "default",
|
||
|
|
"split": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"samples": null,
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": true,
|
||
|
|
"doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
|
||
|
|
"metadata": {
|
||
|
|
"version": 1.0,
|
||
|
|
"pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/",
|
||
|
|
"dtype": "float",
|
||
|
|
"config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/transitive.yaml"
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"blimp_wh_island": {
|
||
|
|
"task": "blimp_wh_island",
|
||
|
|
"dataset_path": "nyu-mll/blimp",
|
||
|
|
"dataset_name": "wh_island",
|
||
|
|
"validation_split": "train",
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"unsafe_code": false,
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"description": "",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "default",
|
||
|
|
"split": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"samples": null,
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": true,
|
||
|
|
"doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
|
||
|
|
"metadata": {
|
||
|
|
"version": 1.0,
|
||
|
|
"pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/",
|
||
|
|
"dtype": "float",
|
||
|
|
"config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/wh_island.yaml"
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"blimp_wh_questions_object_gap": {
|
||
|
|
"task": "blimp_wh_questions_object_gap",
|
||
|
|
"dataset_path": "nyu-mll/blimp",
|
||
|
|
"dataset_name": "wh_questions_object_gap",
|
||
|
|
"validation_split": "train",
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"unsafe_code": false,
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"description": "",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "default",
|
||
|
|
"split": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"samples": null,
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": true,
|
||
|
|
"doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
|
||
|
|
"metadata": {
|
||
|
|
"version": 1.0,
|
||
|
|
"pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/",
|
||
|
|
"dtype": "float",
|
||
|
|
"config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/wh_questions_object_gap.yaml"
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"blimp_wh_questions_subject_gap": {
|
||
|
|
"task": "blimp_wh_questions_subject_gap",
|
||
|
|
"dataset_path": "nyu-mll/blimp",
|
||
|
|
"dataset_name": "wh_questions_subject_gap",
|
||
|
|
"validation_split": "train",
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"unsafe_code": false,
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"description": "",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "default",
|
||
|
|
"split": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"samples": null,
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": true,
|
||
|
|
"doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
|
||
|
|
"metadata": {
|
||
|
|
"version": 1.0,
|
||
|
|
"pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/",
|
||
|
|
"dtype": "float",
|
||
|
|
"config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/wh_questions_subject_gap.yaml"
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"blimp_wh_questions_subject_gap_long_distance": {
|
||
|
|
"task": "blimp_wh_questions_subject_gap_long_distance",
|
||
|
|
"dataset_path": "nyu-mll/blimp",
|
||
|
|
"dataset_name": "wh_questions_subject_gap_long_distance",
|
||
|
|
"validation_split": "train",
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"unsafe_code": false,
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"description": "",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "default",
|
||
|
|
"split": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"samples": null,
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": true,
|
||
|
|
"doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
|
||
|
|
"metadata": {
|
||
|
|
"version": 1.0,
|
||
|
|
"pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/",
|
||
|
|
"dtype": "float",
|
||
|
|
"config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/wh_questions_subject_gap_long_distance.yaml"
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"blimp_wh_vs_that_no_gap": {
|
||
|
|
"task": "blimp_wh_vs_that_no_gap",
|
||
|
|
"dataset_path": "nyu-mll/blimp",
|
||
|
|
"dataset_name": "wh_vs_that_no_gap",
|
||
|
|
"validation_split": "train",
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"unsafe_code": false,
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"description": "",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "default",
|
||
|
|
"split": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"samples": null,
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": true,
|
||
|
|
"doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
|
||
|
|
"metadata": {
|
||
|
|
"version": 1.0,
|
||
|
|
"pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/",
|
||
|
|
"dtype": "float",
|
||
|
|
"config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/wh_vs_that_no_gap.yaml"
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"blimp_wh_vs_that_no_gap_long_distance": {
|
||
|
|
"task": "blimp_wh_vs_that_no_gap_long_distance",
|
||
|
|
"dataset_path": "nyu-mll/blimp",
|
||
|
|
"dataset_name": "wh_vs_that_no_gap_long_distance",
|
||
|
|
"validation_split": "train",
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"unsafe_code": false,
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"description": "",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "default",
|
||
|
|
"split": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"samples": null,
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": true,
|
||
|
|
"doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
|
||
|
|
"metadata": {
|
||
|
|
"version": 1.0,
|
||
|
|
"pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/",
|
||
|
|
"dtype": "float",
|
||
|
|
"config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/wh_vs_that_no_gap_long_distance.yaml"
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"blimp_wh_vs_that_with_gap": {
|
||
|
|
"task": "blimp_wh_vs_that_with_gap",
|
||
|
|
"dataset_path": "nyu-mll/blimp",
|
||
|
|
"dataset_name": "wh_vs_that_with_gap",
|
||
|
|
"validation_split": "train",
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"unsafe_code": false,
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"description": "",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "default",
|
||
|
|
"split": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"samples": null,
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": true,
|
||
|
|
"doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
|
||
|
|
"metadata": {
|
||
|
|
"version": 1.0,
|
||
|
|
"pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/",
|
||
|
|
"dtype": "float",
|
||
|
|
"config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/wh_vs_that_with_gap.yaml"
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"blimp_wh_vs_that_with_gap_long_distance": {
|
||
|
|
"task": "blimp_wh_vs_that_with_gap_long_distance",
|
||
|
|
"dataset_path": "nyu-mll/blimp",
|
||
|
|
"dataset_name": "wh_vs_that_with_gap_long_distance",
|
||
|
|
"validation_split": "train",
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"unsafe_code": false,
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"description": "",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "default",
|
||
|
|
"split": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"samples": null,
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_choice": "{{[sentence_good, sentence_bad]}}",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": true,
|
||
|
|
"doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
|
||
|
|
"metadata": {
|
||
|
|
"version": 1.0,
|
||
|
|
"pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/",
|
||
|
|
"dtype": "float",
|
||
|
|
"config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/wh_vs_that_with_gap_long_distance.yaml"
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"boolq": {
|
||
|
|
"task": "boolq",
|
||
|
|
"dataset_path": "aps/super_glue",
|
||
|
|
"dataset_name": "boolq",
|
||
|
|
"training_split": "train",
|
||
|
|
"validation_split": "validation",
|
||
|
|
"doc_to_text": "{{passage}}\nQuestion: {{question}}?\nAnswer:",
|
||
|
|
"doc_to_target": "label",
|
||
|
|
"unsafe_code": false,
|
||
|
|
"doc_to_choice": [
|
||
|
|
"no",
|
||
|
|
"yes"
|
||
|
|
],
|
||
|
|
"description": "",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "default",
|
||
|
|
"split": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"samples": null,
|
||
|
|
"doc_to_text": "{{passage}}\nQuestion: {{question}}?\nAnswer:",
|
||
|
|
"doc_to_choice": [
|
||
|
|
"no",
|
||
|
|
"yes"
|
||
|
|
],
|
||
|
|
"doc_to_target": "label",
|
||
|
|
"gen_prefix": null,
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc"
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": true,
|
||
|
|
"doc_to_decontamination_query": "passage",
|
||
|
|
"metadata": {
|
||
|
|
"version": 2.0,
|
||
|
|
"pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/",
|
||
|
|
"dtype": "float",
|
||
|
|
"config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/super_glue/boolq/default.yaml"
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"hellaswag": {
|
||
|
|
"task": "hellaswag",
|
||
|
|
"dataset_path": "Rowan/hellaswag",
|
||
|
|
"training_split": "train",
|
||
|
|
"validation_split": "validation",
|
||
|
|
"process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n out_doc = {\n \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n \"gold\": int(doc[\"label\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n",
|
||
|
|
"doc_to_text": "{{query}}",
|
||
|
|
"doc_to_target": "{{label}}",
|
||
|
|
"unsafe_code": false,
|
||
|
|
"doc_to_choice": "choices",
|
||
|
|
"description": "",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "default",
|
||
|
|
"split": null,
|
||
|
|
"process_docs": "<function process_docs at 0x71e28d5e5940>",
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"samples": null,
|
||
|
|
"doc_to_text": "{{query}}",
|
||
|
|
"doc_to_choice": "choices",
|
||
|
|
"doc_to_target": "{{label}}",
|
||
|
|
"gen_prefix": null,
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"metric": "acc_norm",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"metadata": {
|
||
|
|
"version": 1.0,
|
||
|
|
"pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/",
|
||
|
|
"dtype": "float",
|
||
|
|
"config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/hellaswag/hellaswag.yaml"
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"piqa": {
|
||
|
|
"task": "piqa",
|
||
|
|
"dataset_path": "baber/piqa",
|
||
|
|
"training_split": "train",
|
||
|
|
"validation_split": "validation",
|
||
|
|
"doc_to_text": "Question: {{goal}}\nAnswer:",
|
||
|
|
"doc_to_target": "label",
|
||
|
|
"unsafe_code": false,
|
||
|
|
"doc_to_choice": "{{[sol1, sol2]}}",
|
||
|
|
"description": "",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "default",
|
||
|
|
"split": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"samples": null,
|
||
|
|
"doc_to_text": "Question: {{goal}}\nAnswer:",
|
||
|
|
"doc_to_choice": "{{[sol1, sol2]}}",
|
||
|
|
"doc_to_target": "label",
|
||
|
|
"gen_prefix": null,
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"metric": "acc_norm",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": true,
|
||
|
|
"doc_to_decontamination_query": "goal",
|
||
|
|
"metadata": {
|
||
|
|
"version": 1.0,
|
||
|
|
"pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/",
|
||
|
|
"dtype": "float",
|
||
|
|
"config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/piqa/piqa.yaml"
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"wikitext": {
|
||
|
|
"task": "wikitext",
|
||
|
|
"dataset_path": "EleutherAI/wikitext_document_level",
|
||
|
|
"dataset_name": "wikitext-2-raw-v1",
|
||
|
|
"training_split": "train",
|
||
|
|
"validation_split": "validation",
|
||
|
|
"test_split": "test",
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_target": "def wikitext_detokenizer(doc):\n string = doc[\"page\"]\n # contractions\n string = string.replace(\"s '\", \"s'\")\n string = re.sub(r\"/' [0-9]/\", r\"/'[0-9]/\", string)\n # number separators\n string = string.replace(\" @-@ \", \"-\")\n string = string.replace(\" @,@ \", \",\")\n string = string.replace(\" @.@ \", \".\")\n # punctuation\n string = string.replace(\" : \", \": \")\n string = string.replace(\" ; \", \"; \")\n string = string.replace(\" . \", \". \")\n string = string.replace(\" ! \", \"! \")\n string = string.replace(\" ? \", \"? \")\n string = string.replace(\" , \", \", \")\n # double brackets\n string = re.sub(r\"\\(\\s*([^\\)]*?)\\s*\\)\", r\"(\\1)\", string)\n string = re.sub(r\"\\[\\s*([^\\]]*?)\\s*\\]\", r\"[\\1]\", string)\n string = re.sub(r\"{\\s*([^}]*?)\\s*}\", r\"{\\1}\", string)\n string = re.sub(r\"\\\"\\s*([^\\\"]*?)\\s*\\\"\", r'\"\\1\"', string)\n string = re.sub(r\"'\\s*([^']*?)\\s*'\", r\"'\\1'\", string)\n # miscellaneous\n string = string.replace(\"= = = =\", \"====\")\n string = string.replace(\"= = =\", \"===\")\n string = string.replace(\"= =\", \"==\")\n string = string.replace(\" \" + chr(176) + \" \", chr(176))\n string = string.replace(\" \\n\", \"\\n\")\n string = string.replace(\"\\n \", \"\\n\")\n string = string.replace(\" N \", \" 1 \")\n string = string.replace(\" 's\", \"'s\")\n\n return string\n",
|
||
|
|
"unsafe_code": false,
|
||
|
|
"process_results": "def process_results(doc, results):\n (loglikelihood,) = results\n # IMPORTANT: wikitext counts number of words in *original doc before detokenization*\n _words = len(re.split(r\"\\s+\", doc[\"page\"]))\n _bytes = len(doc[\"page\"].encode(\"utf-8\"))\n return {\n \"word_perplexity\": (loglikelihood, _words),\n \"byte_perplexity\": (loglikelihood, _bytes),\n \"bits_per_byte\": (loglikelihood, _bytes),\n }\n",
|
||
|
|
"description": "",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "default",
|
||
|
|
"split": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"samples": null,
|
||
|
|
"doc_to_text": "",
|
||
|
|
"doc_to_choice": null,
|
||
|
|
"doc_to_target": "<function wikitext_detokenizer at 0x71e24ad87920>",
|
||
|
|
"gen_prefix": null,
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "word_perplexity"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"metric": "byte_perplexity"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"metric": "bits_per_byte"
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "loglikelihood_rolling",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": true,
|
||
|
|
"doc_to_decontamination_query": "{{page}}",
|
||
|
|
"metadata": {
|
||
|
|
"version": 2.0,
|
||
|
|
"pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/",
|
||
|
|
"dtype": "float",
|
||
|
|
"config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/wikitext/wikitext.yaml"
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"winogrande": {
|
||
|
|
"task": "winogrande",
|
||
|
|
"dataset_path": "allenai/winogrande",
|
||
|
|
"dataset_name": "winogrande_xl",
|
||
|
|
"training_split": "train",
|
||
|
|
"validation_split": "validation",
|
||
|
|
"doc_to_text": "def doc_to_text(doc):\n answer_to_num = {\"1\": 0, \"2\": 1}\n return answer_to_num[doc[\"answer\"]]\n",
|
||
|
|
"doc_to_target": "def doc_to_target(doc):\n idx = doc[\"sentence\"].index(\"_\") + 1\n return doc[\"sentence\"][idx:].strip()\n",
|
||
|
|
"unsafe_code": false,
|
||
|
|
"doc_to_choice": "def doc_to_choice(doc):\n idx = doc[\"sentence\"].index(\"_\")\n options = [doc[\"option1\"], doc[\"option2\"]]\n return [doc[\"sentence\"][:idx] + opt for opt in options]\n",
|
||
|
|
"description": "",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "default",
|
||
|
|
"split": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"samples": null,
|
||
|
|
"doc_to_text": "<function doc_to_text at 0x71e298d17a60>",
|
||
|
|
"doc_to_choice": "<function doc_to_choice at 0x71e298d179c0>",
|
||
|
|
"doc_to_target": "<function doc_to_target at 0x71e298d17880>",
|
||
|
|
"gen_prefix": null,
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": true,
|
||
|
|
"doc_to_decontamination_query": "sentence",
|
||
|
|
"metadata": {
|
||
|
|
"version": 1.0,
|
||
|
|
"pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/",
|
||
|
|
"dtype": "float",
|
||
|
|
"config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/winogrande/default.yaml"
|
||
|
|
}
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"versions": {
|
||
|
|
"arc_challenge": 1.0,
|
||
|
|
"arc_easy": 1.0,
|
||
|
|
"blimp": "2.0",
|
||
|
|
"blimp_adjunct_island": 1.0,
|
||
|
|
"blimp_anaphor_gender_agreement": 1.0,
|
||
|
|
"blimp_anaphor_number_agreement": 1.0,
|
||
|
|
"blimp_animate_subject_passive": 1.0,
|
||
|
|
"blimp_animate_subject_trans": 1.0,
|
||
|
|
"blimp_causative": 1.0,
|
||
|
|
"blimp_complex_NP_island": 1.0,
|
||
|
|
"blimp_coordinate_structure_constraint_complex_left_branch": 1.0,
|
||
|
|
"blimp_coordinate_structure_constraint_object_extraction": 1.0,
|
||
|
|
"blimp_determiner_noun_agreement_1": 1.0,
|
||
|
|
"blimp_determiner_noun_agreement_2": 1.0,
|
||
|
|
"blimp_determiner_noun_agreement_irregular_1": 1.0,
|
||
|
|
"blimp_determiner_noun_agreement_irregular_2": 1.0,
|
||
|
|
"blimp_determiner_noun_agreement_with_adj_2": 1.0,
|
||
|
|
"blimp_determiner_noun_agreement_with_adj_irregular_1": 1.0,
|
||
|
|
"blimp_determiner_noun_agreement_with_adj_irregular_2": 1.0,
|
||
|
|
"blimp_determiner_noun_agreement_with_adjective_1": 1.0,
|
||
|
|
"blimp_distractor_agreement_relational_noun": 1.0,
|
||
|
|
"blimp_distractor_agreement_relative_clause": 1.0,
|
||
|
|
"blimp_drop_argument": 1.0,
|
||
|
|
"blimp_ellipsis_n_bar_1": 1.0,
|
||
|
|
"blimp_ellipsis_n_bar_2": 1.0,
|
||
|
|
"blimp_existential_there_object_raising": 1.0,
|
||
|
|
"blimp_existential_there_quantifiers_1": 1.0,
|
||
|
|
"blimp_existential_there_quantifiers_2": 1.0,
|
||
|
|
"blimp_existential_there_subject_raising": 1.0,
|
||
|
|
"blimp_expletive_it_object_raising": 1.0,
|
||
|
|
"blimp_inchoative": 1.0,
|
||
|
|
"blimp_intransitive": 1.0,
|
||
|
|
"blimp_irregular_past_participle_adjectives": 1.0,
|
||
|
|
"blimp_irregular_past_participle_verbs": 1.0,
|
||
|
|
"blimp_irregular_plural_subject_verb_agreement_1": 1.0,
|
||
|
|
"blimp_irregular_plural_subject_verb_agreement_2": 1.0,
|
||
|
|
"blimp_left_branch_island_echo_question": 1.0,
|
||
|
|
"blimp_left_branch_island_simple_question": 1.0,
|
||
|
|
"blimp_matrix_question_npi_licensor_present": 1.0,
|
||
|
|
"blimp_npi_present_1": 1.0,
|
||
|
|
"blimp_npi_present_2": 1.0,
|
||
|
|
"blimp_only_npi_licensor_present": 1.0,
|
||
|
|
"blimp_only_npi_scope": 1.0,
|
||
|
|
"blimp_passive_1": 1.0,
|
||
|
|
"blimp_passive_2": 1.0,
|
||
|
|
"blimp_principle_A_c_command": 1.0,
|
||
|
|
"blimp_principle_A_case_1": 1.0,
|
||
|
|
"blimp_principle_A_case_2": 1.0,
|
||
|
|
"blimp_principle_A_domain_1": 1.0,
|
||
|
|
"blimp_principle_A_domain_2": 1.0,
|
||
|
|
"blimp_principle_A_domain_3": 1.0,
|
||
|
|
"blimp_principle_A_reconstruction": 1.0,
|
||
|
|
"blimp_regular_plural_subject_verb_agreement_1": 1.0,
|
||
|
|
"blimp_regular_plural_subject_verb_agreement_2": 1.0,
|
||
|
|
"blimp_sentential_negation_npi_licensor_present": 1.0,
|
||
|
|
"blimp_sentential_negation_npi_scope": 1.0,
|
||
|
|
"blimp_sentential_subject_island": 1.0,
|
||
|
|
"blimp_superlative_quantifiers_1": 1.0,
|
||
|
|
"blimp_superlative_quantifiers_2": 1.0,
|
||
|
|
"blimp_tough_vs_raising_1": 1.0,
|
||
|
|
"blimp_tough_vs_raising_2": 1.0,
|
||
|
|
"blimp_transitive": 1.0,
|
||
|
|
"blimp_wh_island": 1.0,
|
||
|
|
"blimp_wh_questions_object_gap": 1.0,
|
||
|
|
"blimp_wh_questions_subject_gap": 1.0,
|
||
|
|
"blimp_wh_questions_subject_gap_long_distance": 1.0,
|
||
|
|
"blimp_wh_vs_that_no_gap": 1.0,
|
||
|
|
"blimp_wh_vs_that_no_gap_long_distance": 1.0,
|
||
|
|
"blimp_wh_vs_that_with_gap": 1.0,
|
||
|
|
"blimp_wh_vs_that_with_gap_long_distance": 1.0,
|
||
|
|
"boolq": 2.0,
|
||
|
|
"hellaswag": 1.0,
|
||
|
|
"piqa": 1.0,
|
||
|
|
"wikitext": 2.0,
|
||
|
|
"winogrande": 1.0
|
||
|
|
},
|
||
|
|
"n-shot": {
|
||
|
|
"arc_challenge": 0,
|
||
|
|
"arc_easy": 0,
|
||
|
|
"blimp": 0,
|
||
|
|
"blimp_adjunct_island": 0,
|
||
|
|
"blimp_anaphor_gender_agreement": 0,
|
||
|
|
"blimp_anaphor_number_agreement": 0,
|
||
|
|
"blimp_animate_subject_passive": 0,
|
||
|
|
"blimp_animate_subject_trans": 0,
|
||
|
|
"blimp_causative": 0,
|
||
|
|
"blimp_complex_NP_island": 0,
|
||
|
|
"blimp_coordinate_structure_constraint_complex_left_branch": 0,
|
||
|
|
"blimp_coordinate_structure_constraint_object_extraction": 0,
|
||
|
|
"blimp_determiner_noun_agreement_1": 0,
|
||
|
|
"blimp_determiner_noun_agreement_2": 0,
|
||
|
|
"blimp_determiner_noun_agreement_irregular_1": 0,
|
||
|
|
"blimp_determiner_noun_agreement_irregular_2": 0,
|
||
|
|
"blimp_determiner_noun_agreement_with_adj_2": 0,
|
||
|
|
"blimp_determiner_noun_agreement_with_adj_irregular_1": 0,
|
||
|
|
"blimp_determiner_noun_agreement_with_adj_irregular_2": 0,
|
||
|
|
"blimp_determiner_noun_agreement_with_adjective_1": 0,
|
||
|
|
"blimp_distractor_agreement_relational_noun": 0,
|
||
|
|
"blimp_distractor_agreement_relative_clause": 0,
|
||
|
|
"blimp_drop_argument": 0,
|
||
|
|
"blimp_ellipsis_n_bar_1": 0,
|
||
|
|
"blimp_ellipsis_n_bar_2": 0,
|
||
|
|
"blimp_existential_there_object_raising": 0,
|
||
|
|
"blimp_existential_there_quantifiers_1": 0,
|
||
|
|
"blimp_existential_there_quantifiers_2": 0,
|
||
|
|
"blimp_existential_there_subject_raising": 0,
|
||
|
|
"blimp_expletive_it_object_raising": 0,
|
||
|
|
"blimp_inchoative": 0,
|
||
|
|
"blimp_intransitive": 0,
|
||
|
|
"blimp_irregular_past_participle_adjectives": 0,
|
||
|
|
"blimp_irregular_past_participle_verbs": 0,
|
||
|
|
"blimp_irregular_plural_subject_verb_agreement_1": 0,
|
||
|
|
"blimp_irregular_plural_subject_verb_agreement_2": 0,
|
||
|
|
"blimp_left_branch_island_echo_question": 0,
|
||
|
|
"blimp_left_branch_island_simple_question": 0,
|
||
|
|
"blimp_matrix_question_npi_licensor_present": 0,
|
||
|
|
"blimp_npi_present_1": 0,
|
||
|
|
"blimp_npi_present_2": 0,
|
||
|
|
"blimp_only_npi_licensor_present": 0,
|
||
|
|
"blimp_only_npi_scope": 0,
|
||
|
|
"blimp_passive_1": 0,
|
||
|
|
"blimp_passive_2": 0,
|
||
|
|
"blimp_principle_A_c_command": 0,
|
||
|
|
"blimp_principle_A_case_1": 0,
|
||
|
|
"blimp_principle_A_case_2": 0,
|
||
|
|
"blimp_principle_A_domain_1": 0,
|
||
|
|
"blimp_principle_A_domain_2": 0,
|
||
|
|
"blimp_principle_A_domain_3": 0,
|
||
|
|
"blimp_principle_A_reconstruction": 0,
|
||
|
|
"blimp_regular_plural_subject_verb_agreement_1": 0,
|
||
|
|
"blimp_regular_plural_subject_verb_agreement_2": 0,
|
||
|
|
"blimp_sentential_negation_npi_licensor_present": 0,
|
||
|
|
"blimp_sentential_negation_npi_scope": 0,
|
||
|
|
"blimp_sentential_subject_island": 0,
|
||
|
|
"blimp_superlative_quantifiers_1": 0,
|
||
|
|
"blimp_superlative_quantifiers_2": 0,
|
||
|
|
"blimp_tough_vs_raising_1": 0,
|
||
|
|
"blimp_tough_vs_raising_2": 0,
|
||
|
|
"blimp_transitive": 0,
|
||
|
|
"blimp_wh_island": 0,
|
||
|
|
"blimp_wh_questions_object_gap": 0,
|
||
|
|
"blimp_wh_questions_subject_gap": 0,
|
||
|
|
"blimp_wh_questions_subject_gap_long_distance": 0,
|
||
|
|
"blimp_wh_vs_that_no_gap": 0,
|
||
|
|
"blimp_wh_vs_that_no_gap_long_distance": 0,
|
||
|
|
"blimp_wh_vs_that_with_gap": 0,
|
||
|
|
"blimp_wh_vs_that_with_gap_long_distance": 0,
|
||
|
|
"boolq": 0,
|
||
|
|
"hellaswag": 0,
|
||
|
|
"piqa": 0,
|
||
|
|
"wikitext": 0,
|
||
|
|
"winogrande": 0
|
||
|
|
},
|
||
|
|
"higher_is_better": {
|
||
|
|
"arc_challenge": {
|
||
|
|
"acc": true,
|
||
|
|
"acc_norm": true
|
||
|
|
},
|
||
|
|
"arc_easy": {
|
||
|
|
"acc": true,
|
||
|
|
"acc_norm": true
|
||
|
|
},
|
||
|
|
"blimp": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"blimp_adjunct_island": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"blimp_anaphor_gender_agreement": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"blimp_anaphor_number_agreement": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"blimp_animate_subject_passive": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"blimp_animate_subject_trans": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"blimp_causative": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"blimp_complex_NP_island": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"blimp_coordinate_structure_constraint_complex_left_branch": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"blimp_coordinate_structure_constraint_object_extraction": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"blimp_determiner_noun_agreement_1": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"blimp_determiner_noun_agreement_2": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"blimp_determiner_noun_agreement_irregular_1": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"blimp_determiner_noun_agreement_irregular_2": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"blimp_determiner_noun_agreement_with_adj_2": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"blimp_determiner_noun_agreement_with_adj_irregular_1": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"blimp_determiner_noun_agreement_with_adj_irregular_2": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"blimp_determiner_noun_agreement_with_adjective_1": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"blimp_distractor_agreement_relational_noun": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"blimp_distractor_agreement_relative_clause": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"blimp_drop_argument": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"blimp_ellipsis_n_bar_1": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"blimp_ellipsis_n_bar_2": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"blimp_existential_there_object_raising": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"blimp_existential_there_quantifiers_1": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"blimp_existential_there_quantifiers_2": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"blimp_existential_there_subject_raising": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"blimp_expletive_it_object_raising": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"blimp_inchoative": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"blimp_intransitive": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"blimp_irregular_past_participle_adjectives": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"blimp_irregular_past_participle_verbs": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"blimp_irregular_plural_subject_verb_agreement_1": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"blimp_irregular_plural_subject_verb_agreement_2": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"blimp_left_branch_island_echo_question": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"blimp_left_branch_island_simple_question": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"blimp_matrix_question_npi_licensor_present": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"blimp_npi_present_1": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"blimp_npi_present_2": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"blimp_only_npi_licensor_present": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"blimp_only_npi_scope": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"blimp_passive_1": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"blimp_passive_2": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"blimp_principle_A_c_command": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"blimp_principle_A_case_1": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"blimp_principle_A_case_2": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"blimp_principle_A_domain_1": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"blimp_principle_A_domain_2": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"blimp_principle_A_domain_3": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"blimp_principle_A_reconstruction": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"blimp_regular_plural_subject_verb_agreement_1": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"blimp_regular_plural_subject_verb_agreement_2": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"blimp_sentential_negation_npi_licensor_present": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"blimp_sentential_negation_npi_scope": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"blimp_sentential_subject_island": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"blimp_superlative_quantifiers_1": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"blimp_superlative_quantifiers_2": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"blimp_tough_vs_raising_1": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"blimp_tough_vs_raising_2": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"blimp_transitive": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"blimp_wh_island": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"blimp_wh_questions_object_gap": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"blimp_wh_questions_subject_gap": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"blimp_wh_questions_subject_gap_long_distance": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"blimp_wh_vs_that_no_gap": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"blimp_wh_vs_that_no_gap_long_distance": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"blimp_wh_vs_that_with_gap": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"blimp_wh_vs_that_with_gap_long_distance": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"boolq": {
|
||
|
|
"acc": true
|
||
|
|
},
|
||
|
|
"hellaswag": {
|
||
|
|
"acc": true,
|
||
|
|
"acc_norm": true
|
||
|
|
},
|
||
|
|
"piqa": {
|
||
|
|
"acc": true,
|
||
|
|
"acc_norm": true
|
||
|
|
},
|
||
|
|
"wikitext": {
|
||
|
|
"word_perplexity": false,
|
||
|
|
"byte_perplexity": false,
|
||
|
|
"bits_per_byte": false
|
||
|
|
},
|
||
|
|
"winogrande": {
|
||
|
|
"acc": true
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"n-samples": {
|
||
|
|
"winogrande": {
|
||
|
|
"original": 1267,
|
||
|
|
"effective": 1267
|
||
|
|
},
|
||
|
|
"arc_easy": {
|
||
|
|
"original": 2376,
|
||
|
|
"effective": 2376
|
||
|
|
},
|
||
|
|
"hellaswag": {
|
||
|
|
"original": 10042,
|
||
|
|
"effective": 10042
|
||
|
|
},
|
||
|
|
"boolq": {
|
||
|
|
"original": 3270,
|
||
|
|
"effective": 3270
|
||
|
|
},
|
||
|
|
"arc_challenge": {
|
||
|
|
"original": 1172,
|
||
|
|
"effective": 1172
|
||
|
|
},
|
||
|
|
"piqa": {
|
||
|
|
"original": 1838,
|
||
|
|
"effective": 1838
|
||
|
|
},
|
||
|
|
"blimp_adjunct_island": {
|
||
|
|
"original": 1000,
|
||
|
|
"effective": 1000
|
||
|
|
},
|
||
|
|
"blimp_anaphor_gender_agreement": {
|
||
|
|
"original": 1000,
|
||
|
|
"effective": 1000
|
||
|
|
},
|
||
|
|
"blimp_anaphor_number_agreement": {
|
||
|
|
"original": 1000,
|
||
|
|
"effective": 1000
|
||
|
|
},
|
||
|
|
"blimp_animate_subject_passive": {
|
||
|
|
"original": 1000,
|
||
|
|
"effective": 1000
|
||
|
|
},
|
||
|
|
"blimp_animate_subject_trans": {
|
||
|
|
"original": 1000,
|
||
|
|
"effective": 1000
|
||
|
|
},
|
||
|
|
"blimp_causative": {
|
||
|
|
"original": 1000,
|
||
|
|
"effective": 1000
|
||
|
|
},
|
||
|
|
"blimp_complex_NP_island": {
|
||
|
|
"original": 1000,
|
||
|
|
"effective": 1000
|
||
|
|
},
|
||
|
|
"blimp_coordinate_structure_constraint_complex_left_branch": {
|
||
|
|
"original": 1000,
|
||
|
|
"effective": 1000
|
||
|
|
},
|
||
|
|
"blimp_coordinate_structure_constraint_object_extraction": {
|
||
|
|
"original": 1000,
|
||
|
|
"effective": 1000
|
||
|
|
},
|
||
|
|
"blimp_determiner_noun_agreement_1": {
|
||
|
|
"original": 1000,
|
||
|
|
"effective": 1000
|
||
|
|
},
|
||
|
|
"blimp_determiner_noun_agreement_2": {
|
||
|
|
"original": 1000,
|
||
|
|
"effective": 1000
|
||
|
|
},
|
||
|
|
"blimp_determiner_noun_agreement_irregular_1": {
|
||
|
|
"original": 1000,
|
||
|
|
"effective": 1000
|
||
|
|
},
|
||
|
|
"blimp_determiner_noun_agreement_irregular_2": {
|
||
|
|
"original": 1000,
|
||
|
|
"effective": 1000
|
||
|
|
},
|
||
|
|
"blimp_determiner_noun_agreement_with_adj_2": {
|
||
|
|
"original": 1000,
|
||
|
|
"effective": 1000
|
||
|
|
},
|
||
|
|
"blimp_determiner_noun_agreement_with_adj_irregular_1": {
|
||
|
|
"original": 1000,
|
||
|
|
"effective": 1000
|
||
|
|
},
|
||
|
|
"blimp_determiner_noun_agreement_with_adj_irregular_2": {
|
||
|
|
"original": 1000,
|
||
|
|
"effective": 1000
|
||
|
|
},
|
||
|
|
"blimp_determiner_noun_agreement_with_adjective_1": {
|
||
|
|
"original": 1000,
|
||
|
|
"effective": 1000
|
||
|
|
},
|
||
|
|
"blimp_distractor_agreement_relational_noun": {
|
||
|
|
"original": 1000,
|
||
|
|
"effective": 1000
|
||
|
|
},
|
||
|
|
"blimp_distractor_agreement_relative_clause": {
|
||
|
|
"original": 1000,
|
||
|
|
"effective": 1000
|
||
|
|
},
|
||
|
|
"blimp_drop_argument": {
|
||
|
|
"original": 1000,
|
||
|
|
"effective": 1000
|
||
|
|
},
|
||
|
|
"blimp_ellipsis_n_bar_1": {
|
||
|
|
"original": 1000,
|
||
|
|
"effective": 1000
|
||
|
|
},
|
||
|
|
"blimp_ellipsis_n_bar_2": {
|
||
|
|
"original": 1000,
|
||
|
|
"effective": 1000
|
||
|
|
},
|
||
|
|
"blimp_existential_there_object_raising": {
|
||
|
|
"original": 1000,
|
||
|
|
"effective": 1000
|
||
|
|
},
|
||
|
|
"blimp_existential_there_quantifiers_1": {
|
||
|
|
"original": 1000,
|
||
|
|
"effective": 1000
|
||
|
|
},
|
||
|
|
"blimp_existential_there_quantifiers_2": {
|
||
|
|
"original": 1000,
|
||
|
|
"effective": 1000
|
||
|
|
},
|
||
|
|
"blimp_existential_there_subject_raising": {
|
||
|
|
"original": 1000,
|
||
|
|
"effective": 1000
|
||
|
|
},
|
||
|
|
"blimp_expletive_it_object_raising": {
|
||
|
|
"original": 1000,
|
||
|
|
"effective": 1000
|
||
|
|
},
|
||
|
|
"blimp_inchoative": {
|
||
|
|
"original": 1000,
|
||
|
|
"effective": 1000
|
||
|
|
},
|
||
|
|
"blimp_intransitive": {
|
||
|
|
"original": 1000,
|
||
|
|
"effective": 1000
|
||
|
|
},
|
||
|
|
"blimp_irregular_past_participle_adjectives": {
|
||
|
|
"original": 1000,
|
||
|
|
"effective": 1000
|
||
|
|
},
|
||
|
|
"blimp_irregular_past_participle_verbs": {
|
||
|
|
"original": 1000,
|
||
|
|
"effective": 1000
|
||
|
|
},
|
||
|
|
"blimp_irregular_plural_subject_verb_agreement_1": {
|
||
|
|
"original": 1000,
|
||
|
|
"effective": 1000
|
||
|
|
},
|
||
|
|
"blimp_irregular_plural_subject_verb_agreement_2": {
|
||
|
|
"original": 1000,
|
||
|
|
"effective": 1000
|
||
|
|
},
|
||
|
|
"blimp_left_branch_island_echo_question": {
|
||
|
|
"original": 1000,
|
||
|
|
"effective": 1000
|
||
|
|
},
|
||
|
|
"blimp_left_branch_island_simple_question": {
|
||
|
|
"original": 1000,
|
||
|
|
"effective": 1000
|
||
|
|
},
|
||
|
|
"blimp_matrix_question_npi_licensor_present": {
|
||
|
|
"original": 1000,
|
||
|
|
"effective": 1000
|
||
|
|
},
|
||
|
|
"blimp_npi_present_1": {
|
||
|
|
"original": 1000,
|
||
|
|
"effective": 1000
|
||
|
|
},
|
||
|
|
"blimp_npi_present_2": {
|
||
|
|
"original": 1000,
|
||
|
|
"effective": 1000
|
||
|
|
},
|
||
|
|
"blimp_only_npi_licensor_present": {
|
||
|
|
"original": 1000,
|
||
|
|
"effective": 1000
|
||
|
|
},
|
||
|
|
"blimp_only_npi_scope": {
|
||
|
|
"original": 1000,
|
||
|
|
"effective": 1000
|
||
|
|
},
|
||
|
|
"blimp_passive_1": {
|
||
|
|
"original": 1000,
|
||
|
|
"effective": 1000
|
||
|
|
},
|
||
|
|
"blimp_passive_2": {
|
||
|
|
"original": 1000,
|
||
|
|
"effective": 1000
|
||
|
|
},
|
||
|
|
"blimp_principle_A_c_command": {
|
||
|
|
"original": 1000,
|
||
|
|
"effective": 1000
|
||
|
|
},
|
||
|
|
"blimp_principle_A_case_1": {
|
||
|
|
"original": 1000,
|
||
|
|
"effective": 1000
|
||
|
|
},
|
||
|
|
"blimp_principle_A_case_2": {
|
||
|
|
"original": 1000,
|
||
|
|
"effective": 1000
|
||
|
|
},
|
||
|
|
"blimp_principle_A_domain_1": {
|
||
|
|
"original": 1000,
|
||
|
|
"effective": 1000
|
||
|
|
},
|
||
|
|
"blimp_principle_A_domain_2": {
|
||
|
|
"original": 1000,
|
||
|
|
"effective": 1000
|
||
|
|
},
|
||
|
|
"blimp_principle_A_domain_3": {
|
||
|
|
"original": 1000,
|
||
|
|
"effective": 1000
|
||
|
|
},
|
||
|
|
"blimp_principle_A_reconstruction": {
|
||
|
|
"original": 1000,
|
||
|
|
"effective": 1000
|
||
|
|
},
|
||
|
|
"blimp_regular_plural_subject_verb_agreement_1": {
|
||
|
|
"original": 1000,
|
||
|
|
"effective": 1000
|
||
|
|
},
|
||
|
|
"blimp_regular_plural_subject_verb_agreement_2": {
|
||
|
|
"original": 1000,
|
||
|
|
"effective": 1000
|
||
|
|
},
|
||
|
|
"blimp_sentential_negation_npi_licensor_present": {
|
||
|
|
"original": 1000,
|
||
|
|
"effective": 1000
|
||
|
|
},
|
||
|
|
"blimp_sentential_negation_npi_scope": {
|
||
|
|
"original": 1000,
|
||
|
|
"effective": 1000
|
||
|
|
},
|
||
|
|
"blimp_sentential_subject_island": {
|
||
|
|
"original": 1000,
|
||
|
|
"effective": 1000
|
||
|
|
},
|
||
|
|
"blimp_superlative_quantifiers_1": {
|
||
|
|
"original": 1000,
|
||
|
|
"effective": 1000
|
||
|
|
},
|
||
|
|
"blimp_superlative_quantifiers_2": {
|
||
|
|
"original": 1000,
|
||
|
|
"effective": 1000
|
||
|
|
},
|
||
|
|
"blimp_tough_vs_raising_1": {
|
||
|
|
"original": 1000,
|
||
|
|
"effective": 1000
|
||
|
|
},
|
||
|
|
"blimp_tough_vs_raising_2": {
|
||
|
|
"original": 1000,
|
||
|
|
"effective": 1000
|
||
|
|
},
|
||
|
|
"blimp_transitive": {
|
||
|
|
"original": 1000,
|
||
|
|
"effective": 1000
|
||
|
|
},
|
||
|
|
"blimp_wh_island": {
|
||
|
|
"original": 1000,
|
||
|
|
"effective": 1000
|
||
|
|
},
|
||
|
|
"blimp_wh_questions_object_gap": {
|
||
|
|
"original": 1000,
|
||
|
|
"effective": 1000
|
||
|
|
},
|
||
|
|
"blimp_wh_questions_subject_gap": {
|
||
|
|
"original": 1000,
|
||
|
|
"effective": 1000
|
||
|
|
},
|
||
|
|
"blimp_wh_questions_subject_gap_long_distance": {
|
||
|
|
"original": 1000,
|
||
|
|
"effective": 1000
|
||
|
|
},
|
||
|
|
"blimp_wh_vs_that_no_gap": {
|
||
|
|
"original": 1000,
|
||
|
|
"effective": 1000
|
||
|
|
},
|
||
|
|
"blimp_wh_vs_that_no_gap_long_distance": {
|
||
|
|
"original": 1000,
|
||
|
|
"effective": 1000
|
||
|
|
},
|
||
|
|
"blimp_wh_vs_that_with_gap": {
|
||
|
|
"original": 1000,
|
||
|
|
"effective": 1000
|
||
|
|
},
|
||
|
|
"blimp_wh_vs_that_with_gap_long_distance": {
|
||
|
|
"original": 1000,
|
||
|
|
"effective": 1000
|
||
|
|
},
|
||
|
|
"wikitext": {
|
||
|
|
"original": 62,
|
||
|
|
"effective": 62
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"config": {
|
||
|
|
"model": "hf",
|
||
|
|
"model_args": {
|
||
|
|
"pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/",
|
||
|
|
"dtype": "float"
|
||
|
|
},
|
||
|
|
"model_num_parameters": 28355072,
|
||
|
|
"model_dtype": "torch.float32",
|
||
|
|
"model_revision": "main",
|
||
|
|
"model_sha": "",
|
||
|
|
"batch_size": "auto:4",
|
||
|
|
"batch_sizes": [
|
||
|
|
40,
|
||
|
|
64,
|
||
|
|
64,
|
||
|
|
64,
|
||
|
|
64
|
||
|
|
],
|
||
|
|
"device": "cuda1",
|
||
|
|
"use_cache": null,
|
||
|
|
"limit": null,
|
||
|
|
"bootstrap_iters": 100000,
|
||
|
|
"gen_kwargs": {},
|
||
|
|
"random_seed": 0,
|
||
|
|
"numpy_seed": 1234,
|
||
|
|
"torch_seed": 1234,
|
||
|
|
"fewshot_seed": 1234
|
||
|
|
},
|
||
|
|
"git_hash": "1dd93108",
|
||
|
|
"date": 1781007562.4607315,
|
||
|
|
"pretty_env_info": "PyTorch version: 2.12.0+cu130\nIs debug build: False\nCUDA used to build PyTorch: 13.0\nROCM used to build PyTorch: N/A\n\nOS: Linux Mint 22.1 (x86_64)\nGCC version: (Ubuntu 13.3.0-6ubuntu2~24.04) 13.3.0\nClang version: Could not collect\nCMake version: version 3.28.3\nLibc version: glibc-2.39\n\nPython version: 3.12.3 (main, Mar 23 2026, 19:04:32) [GCC 13.3.0] (64-bit runtime)\nPython platform: Linux-6.8.0-51-generic-x86_64-with-glibc2.39\nIs CUDA available: True\nCUDA runtime version: 12.0.140\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA GeForce RTX 3060\nGPU 1: NVIDIA GeForce RTX 2080 Ti\n\nNvidia driver version: 595.71.05\ncuDNN version: Could not collect\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\nCaching allocator config: N/A\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 36\nOn-line CPU(s) list: 0-35\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) CPU E5-2695 v4 @ 2.10GHz\nCPU family: 6\nModel: 79\nThread(s) per core: 2\nCore(s) per socket: 18\nSocket(s): 1\nStepping: 1\nCPU(s) scaling MHz: 45%\nCPU max MHz: 3300,0000\nCPU min MHz: 1200,0000\nBogoMIPS: 4190,53\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cdp_l3 pti intel_ppin ssbd ibrs ibpb stibp tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm cqm rdt_a rdseed adx smap intel_pt xsaveopt cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local dtherm ida arat pln pts vnmi md_clear flush_l1d\nVirtualization: VT-x\nL1d cache: 576 KiB (18 instances)\nL1i cache: 576 KiB (18 instances)\nL2 cache: 4,5 MiB (18 instances)\nL3 cache: 45 MiB (1 instance)\nNUMA node(s): 1\nNUMA node0 CPU(s): 0-35\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit: KVM: Mitigation: VMX disabled\nVulnerability L1tf: Mitigation; PTE Inversion; VMX conditional cache flushes, SMT vulnerable\nVulnerability Mds: Mitigation; Clear CPU buffers; SMT vulnerable\nVulnerability Meltdown: Mitigation; PTI\nVulnerability Mmio stale data: Mitigation; Clear CPU buffers; SMT vulnerable\nVulnerability Reg file data sampling: Not affected\nVulnerability Retbleed: Not affected\nVulnerability Spec rstack overflow: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; IBPB conditional; IBRS_FW; STIBP conditional; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Mitigation; Clear CPU buffers; SMT vulnerable\n\nVersions of relevant libraries:\n[pip3] numpy==2.4.6\n[pip3] nvidia-
|
||
|
|
"transformers_version": "5.10.2",
|
||
|
|
"lm_eval_version": "0.4.13.dev0",
|
||
|
|
"upper_git_hash": null,
|
||
|
|
"tokenizer_pad_token": [
|
||
|
|
"<pad>",
|
||
|
|
"2"
|
||
|
|
],
|
||
|
|
"tokenizer_eos_token": [
|
||
|
|
"<|im_end|>",
|
||
|
|
"1"
|
||
|
|
],
|
||
|
|
"tokenizer_bos_token": [
|
||
|
|
"<s>",
|
||
|
|
"3"
|
||
|
|
],
|
||
|
|
"eot_token_id": 1,
|
||
|
|
"max_length": 1024,
|
||
|
|
"task_hashes": {},
|
||
|
|
"model_source": "hf",
|
||
|
|
"model_name": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/",
|
||
|
|
"model_name_sanitized": "__home__nvidia__pretrain__michel-micro__checkpoint-80000__",
|
||
|
|
"system_instruction": null,
|
||
|
|
"system_instruction_sha": null,
|
||
|
|
"fewshot_as_multiturn": null,
|
||
|
|
"chat_template": null,
|
||
|
|
"chat_template_sha": null,
|
||
|
|
"total_evaluation_time_seconds": "797.8711924949894"
|
||
|
|
}
|