{ "results": { "winogrande": { "name": "winogrande", "alias": "winogrande", "sample_len": 1267, "acc,none": 0.49171270718232046, "acc_stderr,none": 0.01405055532282417 }, "arc_easy": { "name": "arc_easy", "alias": "arc_easy", "sample_len": 2376, "acc,none": 0.4090909090909091, "acc_stderr,none": 0.010088775152615642, "acc_norm,none": 0.3859427609427609, "acc_norm_stderr,none": 0.009989277329504026 }, "hellaswag": { "name": "hellaswag", "alias": "hellaswag", "sample_len": 10042, "acc,none": 0.2749452300338578, "acc_stderr,none": 0.004455741817862, "acc_norm,none": 0.28231428002389963, "acc_norm_stderr,none": 0.004492055279407451 }, "boolq": { "name": "boolq", "alias": "boolq", "sample_len": 3270, "acc,none": 0.5850152905198777, "acc_stderr,none": 0.008617716361921567 }, "arc_challenge": { "name": "arc_challenge", "alias": "arc_challenge", "sample_len": 1172, "acc,none": 0.19795221843003413, "acc_stderr,none": 0.011643990971573369, "acc_norm,none": 0.23122866894197952, "acc_norm_stderr,none": 0.01232085883477234 }, "piqa": { "name": "piqa", "alias": "piqa", "sample_len": 1838, "acc,none": 0.5963003264417845, "acc_stderr,none": 0.011447407541749126, "acc_norm,none": 0.5761697497279652, "acc_norm_stderr,none": 0.011529663270276148 }, "blimp_adjunct_island": { "name": "blimp_adjunct_island", "alias": "blimp_adjunct_island", "sample_len": 1000, "acc,none": 0.755, "acc_stderr,none": 0.01360735683959821 }, "blimp_anaphor_gender_agreement": { "name": "blimp_anaphor_gender_agreement", "alias": "blimp_anaphor_gender_agreement", "sample_len": 1000, "acc,none": 0.844, "acc_stderr,none": 0.011480235006122295 }, "blimp_anaphor_number_agreement": { "name": "blimp_anaphor_number_agreement", "alias": "blimp_anaphor_number_agreement", "sample_len": 1000, "acc,none": 0.909, "acc_stderr,none": 0.009099549538400337 }, "blimp_animate_subject_passive": { "name": "blimp_animate_subject_passive", "alias": "blimp_animate_subject_passive", "sample_len": 1000, "acc,none": 0.719, "acc_stderr,none": 0.01422115470843494 }, "blimp_animate_subject_trans": { "name": "blimp_animate_subject_trans", "alias": "blimp_animate_subject_trans", "sample_len": 1000, "acc,none": 0.726, "acc_stderr,none": 0.01411109928825969 }, "blimp_causative": { "name": "blimp_causative", "alias": "blimp_causative", "sample_len": 1000, "acc,none": 0.533, "acc_stderr,none": 0.015784807891138876 }, "blimp_complex_NP_island": { "name": "blimp_complex_NP_island", "alias": "blimp_complex_NP_island", "sample_len": 1000, "acc,none": 0.447, "acc_stderr,none": 0.015730176046009098 }, "blimp_coordinate_structure_constraint_complex_left_branch": { "name": "blimp_coordinate_structure_constraint_complex_left_branch", "alias": "blimp_coordinate_structure_constraint_complex_left_branch", "sample_len": 1000, "acc,none": 0.672, "acc_stderr,none": 0.01485384248727031 }, "blimp_coordinate_structure_constraint_object_extraction": { "name": "blimp_coordinate_structure_constraint_object_extraction", "alias": "blimp_coordinate_structure_constraint_object_extraction", "sample_len": 1000, "acc,none": 0.581, "acc_stderr,none": 0.015610338967577953 }, "blimp_determiner_noun_agreement_1": { "name": "blimp_determiner_noun_agreement_1", "alias": "blimp_determiner_noun_agreement_1", "sample_len": 1000, "acc,none": 0.925, "acc_stderr,none": 0.008333333333333333 }, "blimp_determiner_noun_agreement_2": { "name": "blimp_determiner_noun_agreement_2", "alias": "blimp_determiner_noun_agreement_2", "sample_len": 1000, "acc,none": 0.919, "acc_stderr,none": 0.008632121032139986 }, "blimp_determiner_noun_agreement_irregular_1": { "name": "blimp_determiner_noun_agreement_irregular_1", "alias": "blimp_determiner_noun_agreement_irregular_1", "sample_len": 1000, "acc,none": 0.796, "acc_stderr,none": 0.012749374359024311 }, "blimp_determiner_noun_agreement_irregular_2": { "name": "blimp_determiner_noun_agreement_irregular_2", "alias": "blimp_determiner_noun_agreement_irregular_2", "sample_len": 1000, "acc,none": 0.904, "acc_stderr,none": 0.009320454434783257 }, "blimp_determiner_noun_agreement_with_adj_2": { "name": "blimp_determiner_noun_agreement_with_adj_2", "alias": "blimp_determiner_noun_agreement_with_adj_2", "sample_len": 1000, "acc,none": 0.888, "acc_stderr,none": 0.009977753031397177 }, "blimp_determiner_noun_agreement_with_adj_irregular_1": { "name": "blimp_determiner_noun_agreement_with_adj_irregular_1", "alias": "blimp_determiner_noun_agreement_with_adj_irregular_1", "sample_len": 1000, "acc,none": 0.804, "acc_stderr,none": 0.012559527926707347 }, "blimp_determiner_noun_agreement_with_adj_irregular_2": { "name": "blimp_determiner_noun_agreement_with_adj_irregular_2", "alias": "blimp_determiner_noun_agreement_with_adj_irregular_2", "sample_len": 1000, "acc,none": 0.827, "acc_stderr,none": 0.011967214137559927 }, "blimp_determiner_noun_agreement_with_adjective_1": { "name": "blimp_determiner_noun_agreement_with_adjective_1", "alias": "blimp_determiner_noun_agreement_with_adjective_1", "sample_len": 1000, "acc,none": 0.892, "acc_stderr,none": 0.009820001651345708 }, "blimp_distractor_agreement_relational_noun": { "name": "blimp_distractor_agreement_relational_noun", "alias": "blimp_distractor_agreement_relational_noun", "sample_len": 1000, "acc,none": 0.57, "acc_stderr,none": 0.01566350361015521 }, "blimp_distractor_agreement_relative_clause": { "name": "blimp_distractor_agreement_relative_clause", "alias": "blimp_distractor_agreement_relative_clause", "sample_len": 1000, "acc,none": 0.477, "acc_stderr,none": 0.015802554246726087 }, "blimp_drop_argument": { "name": "blimp_drop_argument", "alias": "blimp_drop_argument", "sample_len": 1000, "acc,none": 0.718, "acc_stderr,none": 0.01423652621529138 }, "blimp_ellipsis_n_bar_1": { "name": "blimp_ellipsis_n_bar_1", "alias": "blimp_ellipsis_n_bar_1", "sample_len": 1000, "acc,none": 0.619, "acc_stderr,none": 0.015364734787007391 }, "blimp_ellipsis_n_bar_2": { "name": "blimp_ellipsis_n_bar_2", "alias": "blimp_ellipsis_n_bar_2", "sample_len": 1000, "acc,none": 0.942, "acc_stderr,none": 0.0073953154557929906 }, "blimp_existential_there_object_raising": { "name": "blimp_existential_there_object_raising", "alias": "blimp_existential_there_object_raising", "sample_len": 1000, "acc,none": 0.841, "acc_stderr,none": 0.011569479368271195 }, "blimp_existential_there_quantifiers_1": { "name": "blimp_existential_there_quantifiers_1", "alias": "blimp_existential_there_quantifiers_1", "sample_len": 1000, "acc,none": 0.959, "acc_stderr,none": 0.006273624021118764 }, "blimp_existential_there_quantifiers_2": { "name": "blimp_existential_there_quantifiers_2", "alias": "blimp_existential_there_quantifiers_2", "sample_len": 1000, "acc,none": 0.442, "acc_stderr,none": 0.015712507211864152 }, "blimp_existential_there_subject_raising": { "name": "blimp_existential_there_subject_raising", "alias": "blimp_existential_there_subject_raising", "sample_len": 1000, "acc,none": 0.78, "acc_stderr,none": 0.013106173040661862 }, "blimp_expletive_it_object_raising": { "name": "blimp_expletive_it_object_raising", "alias": "blimp_expletive_it_object_raising", "sample_len": 1000, "acc,none": 0.687, "acc_stderr,none": 0.014671272822977775 }, "blimp_inchoative": { "name": "blimp_inchoative", "alias": "blimp_inchoative", "sample_len": 1000, "acc,none": 0.446, "acc_stderr,none": 0.01572677116675039 }, "blimp_intransitive": { "name": "blimp_intransitive", "alias": "blimp_intransitive", "sample_len": 1000, "acc,none": 0.575, "acc_stderr,none": 0.01564032031704017 }, "blimp_irregular_past_participle_adjectives": { "name": "blimp_irregular_past_participle_adjectives", "alias": "blimp_irregular_past_participle_adjectives", "sample_len": 1000, "acc,none": 0.801, "acc_stderr,none": 0.01263164908309911 }, "blimp_irregular_past_participle_verbs": { "name": "blimp_irregular_past_participle_verbs", "alias": "blimp_irregular_past_participle_verbs", "sample_len": 1000, "acc,none": 0.694, "acc_stderr,none": 0.014580006055436898 }, "blimp_irregular_plural_subject_verb_agreement_1": { "name": "blimp_irregular_plural_subject_verb_agreement_1", "alias": "blimp_irregular_plural_subject_verb_agreement_1", "sample_len": 1000, "acc,none": 0.781, "acc_stderr,none": 0.013084731950262116 }, "blimp_irregular_plural_subject_verb_agreement_2": { "name": "blimp_irregular_plural_subject_verb_agreement_2", "alias": "blimp_irregular_plural_subject_verb_agreement_2", "sample_len": 1000, "acc,none": 0.819, "acc_stderr,none": 0.012181436179178027 }, "blimp_left_branch_island_echo_question": { "name": "blimp_left_branch_island_echo_question", "alias": "blimp_left_branch_island_echo_question", "sample_len": 1000, "acc,none": 0.656, "acc_stderr,none": 0.015029633724408919 }, "blimp_left_branch_island_simple_question": { "name": "blimp_left_branch_island_simple_question", "alias": "blimp_left_branch_island_simple_question", "sample_len": 1000, "acc,none": 0.625, "acc_stderr,none": 0.015316971293620996 }, "blimp_matrix_question_npi_licensor_present": { "name": "blimp_matrix_question_npi_licensor_present", "alias": "blimp_matrix_question_npi_licensor_present", "sample_len": 1000, "acc,none": 0.074, "acc_stderr,none": 0.0082820645127041 }, "blimp_npi_present_1": { "name": "blimp_npi_present_1", "alias": "blimp_npi_present_1", "sample_len": 1000, "acc,none": 0.341, "acc_stderr,none": 0.01499813134840284 }, "blimp_npi_present_2": { "name": "blimp_npi_present_2", "alias": "blimp_npi_present_2", "sample_len": 1000, "acc,none": 0.262, "acc_stderr,none": 0.013912208651021217 }, "blimp_only_npi_licensor_present": { "name": "blimp_only_npi_licensor_present", "alias": "blimp_only_npi_licensor_present", "sample_len": 1000, "acc,none": 0.875, "acc_stderr,none": 0.010463483381956722 }, "blimp_only_npi_scope": { "name": "blimp_only_npi_scope", "alias": "blimp_only_npi_scope", "sample_len": 1000, "acc,none": 0.922, "acc_stderr,none": 0.00848457353011853 }, "blimp_passive_1": { "name": "blimp_passive_1", "alias": "blimp_passive_1", "sample_len": 1000, "acc,none": 0.778, "acc_stderr,none": 0.013148721948877349 }, "blimp_passive_2": { "name": "blimp_passive_2", "alias": "blimp_passive_2", "sample_len": 1000, "acc,none": 0.713, "acc_stderr,none": 0.014312087053810001 }, "blimp_principle_A_c_command": { "name": "blimp_principle_A_c_command", "alias": "blimp_principle_A_c_command", "sample_len": 1000, "acc,none": 0.563, "acc_stderr,none": 0.01569322392873043 }, "blimp_principle_A_case_1": { "name": "blimp_principle_A_case_1", "alias": "blimp_principle_A_case_1", "sample_len": 1000, "acc,none": 0.999, "acc_stderr,none": 0.001 }, "blimp_principle_A_case_2": { "name": "blimp_principle_A_case_2", "alias": "blimp_principle_A_case_2", "sample_len": 1000, "acc,none": 0.801, "acc_stderr,none": 0.01263164908309911 }, "blimp_principle_A_domain_1": { "name": "blimp_principle_A_domain_1", "alias": "blimp_principle_A_domain_1", "sample_len": 1000, "acc,none": 0.94, "acc_stderr,none": 0.007513751157474971 }, "blimp_principle_A_domain_2": { "name": "blimp_principle_A_domain_2", "alias": "blimp_principle_A_domain_2", "sample_len": 1000, "acc,none": 0.713, "acc_stderr,none": 0.014312087053810001 }, "blimp_principle_A_domain_3": { "name": "blimp_principle_A_domain_3", "alias": "blimp_principle_A_domain_3", "sample_len": 1000, "acc,none": 0.535, "acc_stderr,none": 0.015780495050030086 }, "blimp_principle_A_reconstruction": { "name": "blimp_principle_A_reconstruction", "alias": "blimp_principle_A_reconstruction", "sample_len": 1000, "acc,none": 0.53, "acc_stderr,none": 0.015790799515836725 }, "blimp_regular_plural_subject_verb_agreement_1": { "name": "blimp_regular_plural_subject_verb_agreement_1", "alias": "blimp_regular_plural_subject_verb_agreement_1", "sample_len": 1000, "acc,none": 0.751, "acc_stderr,none": 0.013681600278702275 }, "blimp_regular_plural_subject_verb_agreement_2": { "name": "blimp_regular_plural_subject_verb_agreement_2", "alias": "blimp_regular_plural_subject_verb_agreement_2", "sample_len": 1000, "acc,none": 0.762, "acc_stderr,none": 0.013473586661967093 }, "blimp_sentential_negation_npi_licensor_present": { "name": "blimp_sentential_negation_npi_licensor_present", "alias": "blimp_sentential_negation_npi_licensor_present", "sample_len": 1000, "acc,none": 0.995, "acc_stderr,none": 0.0022315868748448895 }, "blimp_sentential_negation_npi_scope": { "name": "blimp_sentential_negation_npi_scope", "alias": "blimp_sentential_negation_npi_scope", "sample_len": 1000, "acc,none": 0.488, "acc_stderr,none": 0.015814743314581686 }, "blimp_sentential_subject_island": { "name": "blimp_sentential_subject_island", "alias": "blimp_sentential_subject_island", "sample_len": 1000, "acc,none": 0.401, "acc_stderr,none": 0.015506109745498368 }, "blimp_superlative_quantifiers_1": { "name": "blimp_superlative_quantifiers_1", "alias": "blimp_superlative_quantifiers_1", "sample_len": 1000, "acc,none": 0.738, "acc_stderr,none": 0.013912208651021217 }, "blimp_superlative_quantifiers_2": { "name": "blimp_superlative_quantifiers_2", "alias": "blimp_superlative_quantifiers_2", "sample_len": 1000, "acc,none": 0.671, "acc_stderr,none": 0.014865395385928355 }, "blimp_tough_vs_raising_1": { "name": "blimp_tough_vs_raising_1", "alias": "blimp_tough_vs_raising_1", "sample_len": 1000, "acc,none": 0.42, "acc_stderr,none": 0.015615500115072968 }, "blimp_tough_vs_raising_2": { "name": "blimp_tough_vs_raising_2", "alias": "blimp_tough_vs_raising_2", "sample_len": 1000, "acc,none": 0.728, "acc_stderr,none": 0.014078856992462642 }, "blimp_transitive": { "name": "blimp_transitive", "alias": "blimp_transitive", "sample_len": 1000, "acc,none": 0.722, "acc_stderr,none": 0.014174516461485272 }, "blimp_wh_island": { "name": "blimp_wh_island", "alias": "blimp_wh_island", "sample_len": 1000, "acc,none": 0.602, "acc_stderr,none": 0.015486634102859016 }, "blimp_wh_questions_object_gap": { "name": "blimp_wh_questions_object_gap", "alias": "blimp_wh_questions_object_gap", "sample_len": 1000, "acc,none": 0.639, "acc_stderr,none": 0.015195720118175049 }, "blimp_wh_questions_subject_gap": { "name": "blimp_wh_questions_subject_gap", "alias": "blimp_wh_questions_subject_gap", "sample_len": 1000, "acc,none": 0.853, "acc_stderr,none": 0.011203415395160376 }, "blimp_wh_questions_subject_gap_long_distance": { "name": "blimp_wh_questions_subject_gap_long_distance", "alias": "blimp_wh_questions_subject_gap_long_distance", "sample_len": 1000, "acc,none": 0.949, "acc_stderr,none": 0.0069604200625714725 }, "blimp_wh_vs_that_no_gap": { "name": "blimp_wh_vs_that_no_gap", "alias": "blimp_wh_vs_that_no_gap", "sample_len": 1000, "acc,none": 0.879, "acc_stderr,none": 0.010318210380946179 }, "blimp_wh_vs_that_no_gap_long_distance": { "name": "blimp_wh_vs_that_no_gap_long_distance", "alias": "blimp_wh_vs_that_no_gap_long_distance", "sample_len": 1000, "acc,none": 0.944, "acc_stderr,none": 0.007274401481697097 }, "blimp_wh_vs_that_with_gap": { "name": "blimp_wh_vs_that_with_gap", "alias": "blimp_wh_vs_that_with_gap", "sample_len": 1000, "acc,none": 0.394, "acc_stderr,none": 0.015459721957493382 }, "blimp_wh_vs_that_with_gap_long_distance": { "name": "blimp_wh_vs_that_with_gap_long_distance", "alias": "blimp_wh_vs_that_with_gap_long_distance", "sample_len": 1000, "acc,none": 0.178, "acc_stderr,none": 0.012102167676183571 }, "wikitext": { "name": "wikitext", "alias": "wikitext", "sample_len": 62, "word_perplexity,none": 86.64177355483464, "word_perplexity_stderr,none": "N/A", "byte_perplexity,none": 2.3033752584676135, "byte_perplexity_stderr,none": "N/A", "bits_per_byte,none": 1.2037494695014286, "bits_per_byte_stderr,none": "N/A" }, "blimp": { "alias": "blimp", "name": "blimp", "sample_len": 67000, "acc,none": 0.6975074626865672, "acc_stderr,none": 0.0015947299360720968, "sample_count": { "acc,none": 67000 } } }, "groups": { "blimp": { "alias": "blimp", "name": "blimp", "sample_len": 67000, "acc,none": 0.6975074626865672, "acc_stderr,none": 0.0015947299360720968, "sample_count": { "acc,none": 67000 } } }, "group_subtasks": { "blimp": [ "blimp_adjunct_island", "blimp_anaphor_gender_agreement", "blimp_anaphor_number_agreement", "blimp_animate_subject_passive", "blimp_animate_subject_trans", "blimp_causative", "blimp_complex_NP_island", "blimp_coordinate_structure_constraint_complex_left_branch", "blimp_coordinate_structure_constraint_object_extraction", "blimp_determiner_noun_agreement_1", "blimp_determiner_noun_agreement_2", "blimp_determiner_noun_agreement_irregular_1", "blimp_determiner_noun_agreement_irregular_2", "blimp_determiner_noun_agreement_with_adj_2", "blimp_determiner_noun_agreement_with_adj_irregular_1", "blimp_determiner_noun_agreement_with_adj_irregular_2", "blimp_determiner_noun_agreement_with_adjective_1", "blimp_distractor_agreement_relational_noun", "blimp_distractor_agreement_relative_clause", "blimp_drop_argument", "blimp_ellipsis_n_bar_1", "blimp_ellipsis_n_bar_2", "blimp_existential_there_object_raising", "blimp_existential_there_quantifiers_1", "blimp_existential_there_quantifiers_2", "blimp_existential_there_subject_raising", "blimp_expletive_it_object_raising", "blimp_inchoative", "blimp_intransitive", "blimp_irregular_past_participle_adjectives", "blimp_irregular_past_participle_verbs", "blimp_irregular_plural_subject_verb_agreement_1", "blimp_irregular_plural_subject_verb_agreement_2", "blimp_left_branch_island_echo_question", "blimp_left_branch_island_simple_question", "blimp_matrix_question_npi_licensor_present", "blimp_npi_present_1", "blimp_npi_present_2", "blimp_only_npi_licensor_present", "blimp_only_npi_scope", "blimp_passive_1", "blimp_passive_2", "blimp_principle_A_c_command", "blimp_principle_A_case_1", "blimp_principle_A_case_2", "blimp_principle_A_domain_1", "blimp_principle_A_domain_2", "blimp_principle_A_domain_3", "blimp_principle_A_reconstruction", "blimp_regular_plural_subject_verb_agreement_1", "blimp_regular_plural_subject_verb_agreement_2", "blimp_sentential_negation_npi_licensor_present", "blimp_sentential_negation_npi_scope", "blimp_sentential_subject_island", "blimp_superlative_quantifiers_1", "blimp_superlative_quantifiers_2", "blimp_tough_vs_raising_1", "blimp_tough_vs_raising_2", "blimp_transitive", "blimp_wh_island", "blimp_wh_questions_object_gap", "blimp_wh_questions_subject_gap", "blimp_wh_questions_subject_gap_long_distance", "blimp_wh_vs_that_no_gap", "blimp_wh_vs_that_no_gap_long_distance", "blimp_wh_vs_that_with_gap", "blimp_wh_vs_that_with_gap_long_distance" ] }, "configs": { "arc_challenge": { "task": "arc_challenge", "dataset_path": "allenai/ai2_arc", "dataset_name": "ARC-Challenge", "training_split": "train", "validation_split": "validation", "test_split": "test", "doc_to_text": "Question: {{question}}\nAnswer:", "doc_to_target": "{{choices.label.index(answerKey)}}", "unsafe_code": false, "doc_to_choice": "{{choices.text}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": null, "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "Question: {{question}}\nAnswer:", "doc_to_choice": "{{choices.text}}", "doc_to_target": "{{choices.label.index(answerKey)}}", "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true }, { "metric": "acc_norm", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "Question: {{question}}\nAnswer:", "metadata": { "version": 1.0, "pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/", "dtype": "float", "config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/arc/arc_challenge.yaml" } }, "arc_easy": { "task": "arc_easy", "dataset_path": "allenai/ai2_arc", "dataset_name": "ARC-Easy", "training_split": "train", "validation_split": "validation", "test_split": "test", "doc_to_text": "Question: {{question}}\nAnswer:", "doc_to_target": "{{choices.label.index(answerKey)}}", "unsafe_code": false, "doc_to_choice": "{{choices.text}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": null, "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "Question: {{question}}\nAnswer:", "doc_to_choice": "{{choices.text}}", "doc_to_target": "{{choices.label.index(answerKey)}}", "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true }, { "metric": "acc_norm", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "Question: {{question}}\nAnswer:", "metadata": { "version": 1.0, "pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/", "dtype": "float", "config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/arc/arc_easy.yaml" } }, "blimp_adjunct_island": { "task": "blimp_adjunct_island", "dataset_path": "nyu-mll/blimp", "dataset_name": "adjunct_island", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "unsafe_code": false, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": null, "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "", "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "doc_to_target": 0, "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0, "pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/", "dtype": "float", "config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/adjunct_island.yaml" } }, "blimp_anaphor_gender_agreement": { "task": "blimp_anaphor_gender_agreement", "dataset_path": "nyu-mll/blimp", "dataset_name": "anaphor_gender_agreement", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "unsafe_code": false, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": null, "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "", "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "doc_to_target": 0, "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0, "pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/", "dtype": "float", "config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/anaphor_gender_agreement.yaml" } }, "blimp_anaphor_number_agreement": { "task": "blimp_anaphor_number_agreement", "dataset_path": "nyu-mll/blimp", "dataset_name": "anaphor_number_agreement", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "unsafe_code": false, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": null, "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "", "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "doc_to_target": 0, "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0, "pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/", "dtype": "float", "config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/anaphor_number_agreement.yaml" } }, "blimp_animate_subject_passive": { "task": "blimp_animate_subject_passive", "dataset_path": "nyu-mll/blimp", "dataset_name": "animate_subject_passive", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "unsafe_code": false, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": null, "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "", "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "doc_to_target": 0, "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0, "pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/", "dtype": "float", "config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/animate_subject_passive.yaml" } }, "blimp_animate_subject_trans": { "task": "blimp_animate_subject_trans", "dataset_path": "nyu-mll/blimp", "dataset_name": "animate_subject_trans", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "unsafe_code": false, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": null, "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "", "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "doc_to_target": 0, "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0, "pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/", "dtype": "float", "config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/animate_subject_trans.yaml" } }, "blimp_causative": { "task": "blimp_causative", "dataset_path": "nyu-mll/blimp", "dataset_name": "causative", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "unsafe_code": false, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": null, "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "", "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "doc_to_target": 0, "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0, "pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/", "dtype": "float", "config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/causative.yaml" } }, "blimp_complex_NP_island": { "task": "blimp_complex_NP_island", "dataset_path": "nyu-mll/blimp", "dataset_name": "complex_NP_island", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "unsafe_code": false, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": null, "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "", "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "doc_to_target": 0, "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0, "pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/", "dtype": "float", "config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/complex_NP_island.yaml" } }, "blimp_coordinate_structure_constraint_complex_left_branch": { "task": "blimp_coordinate_structure_constraint_complex_left_branch", "dataset_path": "nyu-mll/blimp", "dataset_name": "coordinate_structure_constraint_complex_left_branch", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "unsafe_code": false, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": null, "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "", "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "doc_to_target": 0, "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0, "pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/", "dtype": "float", "config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/coordinate_structure_constraint_complex_left_branch.yaml" } }, "blimp_coordinate_structure_constraint_object_extraction": { "task": "blimp_coordinate_structure_constraint_object_extraction", "dataset_path": "nyu-mll/blimp", "dataset_name": "coordinate_structure_constraint_object_extraction", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "unsafe_code": false, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": null, "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "", "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "doc_to_target": 0, "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0, "pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/", "dtype": "float", "config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/coordinate_structure_constraint_object_extraction.yaml" } }, "blimp_determiner_noun_agreement_1": { "task": "blimp_determiner_noun_agreement_1", "dataset_path": "nyu-mll/blimp", "dataset_name": "determiner_noun_agreement_1", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "unsafe_code": false, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": null, "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "", "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "doc_to_target": 0, "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0, "pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/", "dtype": "float", "config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/determiner_noun_agreement_1.yaml" } }, "blimp_determiner_noun_agreement_2": { "task": "blimp_determiner_noun_agreement_2", "dataset_path": "nyu-mll/blimp", "dataset_name": "determiner_noun_agreement_2", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "unsafe_code": false, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": null, "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "", "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "doc_to_target": 0, "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0, "pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/", "dtype": "float", "config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/determiner_noun_agreement_2.yaml" } }, "blimp_determiner_noun_agreement_irregular_1": { "task": "blimp_determiner_noun_agreement_irregular_1", "dataset_path": "nyu-mll/blimp", "dataset_name": "determiner_noun_agreement_irregular_1", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "unsafe_code": false, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": null, "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "", "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "doc_to_target": 0, "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0, "pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/", "dtype": "float", "config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_1.yaml" } }, "blimp_determiner_noun_agreement_irregular_2": { "task": "blimp_determiner_noun_agreement_irregular_2", "dataset_path": "nyu-mll/blimp", "dataset_name": "determiner_noun_agreement_irregular_2", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "unsafe_code": false, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": null, "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "", "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "doc_to_target": 0, "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0, "pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/", "dtype": "float", "config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_2.yaml" } }, "blimp_determiner_noun_agreement_with_adj_2": { "task": "blimp_determiner_noun_agreement_with_adj_2", "dataset_path": "nyu-mll/blimp", "dataset_name": "determiner_noun_agreement_with_adj_2", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "unsafe_code": false, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": null, "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "", "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "doc_to_target": 0, "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0, "pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/", "dtype": "float", "config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_2.yaml" } }, "blimp_determiner_noun_agreement_with_adj_irregular_1": { "task": "blimp_determiner_noun_agreement_with_adj_irregular_1", "dataset_path": "nyu-mll/blimp", "dataset_name": "determiner_noun_agreement_with_adj_irregular_1", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "unsafe_code": false, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": null, "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "", "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "doc_to_target": 0, "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0, "pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/", "dtype": "float", "config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_1.yaml" } }, "blimp_determiner_noun_agreement_with_adj_irregular_2": { "task": "blimp_determiner_noun_agreement_with_adj_irregular_2", "dataset_path": "nyu-mll/blimp", "dataset_name": "determiner_noun_agreement_with_adj_irregular_2", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "unsafe_code": false, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": null, "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "", "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "doc_to_target": 0, "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0, "pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/", "dtype": "float", "config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_2.yaml" } }, "blimp_determiner_noun_agreement_with_adjective_1": { "task": "blimp_determiner_noun_agreement_with_adjective_1", "dataset_path": "nyu-mll/blimp", "dataset_name": "determiner_noun_agreement_with_adjective_1", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "unsafe_code": false, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": null, "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "", "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "doc_to_target": 0, "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0, "pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/", "dtype": "float", "config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/determiner_noun_agreement_with_adjective_1.yaml" } }, "blimp_distractor_agreement_relational_noun": { "task": "blimp_distractor_agreement_relational_noun", "dataset_path": "nyu-mll/blimp", "dataset_name": "distractor_agreement_relational_noun", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "unsafe_code": false, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": null, "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "", "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "doc_to_target": 0, "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0, "pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/", "dtype": "float", "config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/distractor_agreement_relational_noun.yaml" } }, "blimp_distractor_agreement_relative_clause": { "task": "blimp_distractor_agreement_relative_clause", "dataset_path": "nyu-mll/blimp", "dataset_name": "distractor_agreement_relative_clause", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "unsafe_code": false, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": null, "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "", "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "doc_to_target": 0, "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0, "pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/", "dtype": "float", "config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/distractor_agreement_relative_clause.yaml" } }, "blimp_drop_argument": { "task": "blimp_drop_argument", "dataset_path": "nyu-mll/blimp", "dataset_name": "drop_argument", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "unsafe_code": false, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": null, "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "", "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "doc_to_target": 0, "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0, "pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/", "dtype": "float", "config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/drop_argument.yaml" } }, "blimp_ellipsis_n_bar_1": { "task": "blimp_ellipsis_n_bar_1", "dataset_path": "nyu-mll/blimp", "dataset_name": "ellipsis_n_bar_1", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "unsafe_code": false, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": null, "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "", "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "doc_to_target": 0, "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0, "pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/", "dtype": "float", "config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/ellipsis_n_bar_1.yaml" } }, "blimp_ellipsis_n_bar_2": { "task": "blimp_ellipsis_n_bar_2", "dataset_path": "nyu-mll/blimp", "dataset_name": "ellipsis_n_bar_2", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "unsafe_code": false, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": null, "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "", "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "doc_to_target": 0, "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0, "pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/", "dtype": "float", "config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/ellipsis_n_bar_2.yaml" } }, "blimp_existential_there_object_raising": { "task": "blimp_existential_there_object_raising", "dataset_path": "nyu-mll/blimp", "dataset_name": "existential_there_object_raising", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "unsafe_code": false, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": null, "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "", "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "doc_to_target": 0, "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0, "pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/", "dtype": "float", "config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/existential_there_object_raising.yaml" } }, "blimp_existential_there_quantifiers_1": { "task": "blimp_existential_there_quantifiers_1", "dataset_path": "nyu-mll/blimp", "dataset_name": "existential_there_quantifiers_1", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "unsafe_code": false, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": null, "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "", "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "doc_to_target": 0, "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0, "pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/", "dtype": "float", "config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/existential_there_quantifiers_1.yaml" } }, "blimp_existential_there_quantifiers_2": { "task": "blimp_existential_there_quantifiers_2", "dataset_path": "nyu-mll/blimp", "dataset_name": "existential_there_quantifiers_2", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "unsafe_code": false, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": null, "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "", "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "doc_to_target": 0, "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0, "pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/", "dtype": "float", "config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/existential_there_quantifiers_2.yaml" } }, "blimp_existential_there_subject_raising": { "task": "blimp_existential_there_subject_raising", "dataset_path": "nyu-mll/blimp", "dataset_name": "existential_there_subject_raising", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "unsafe_code": false, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": null, "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "", "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "doc_to_target": 0, "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0, "pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/", "dtype": "float", "config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/existential_there_subject_raising.yaml" } }, "blimp_expletive_it_object_raising": { "task": "blimp_expletive_it_object_raising", "dataset_path": "nyu-mll/blimp", "dataset_name": "expletive_it_object_raising", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "unsafe_code": false, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": null, "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "", "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "doc_to_target": 0, "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0, "pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/", "dtype": "float", "config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/expletive_it_object_raising.yaml" } }, "blimp_inchoative": { "task": "blimp_inchoative", "dataset_path": "nyu-mll/blimp", "dataset_name": "inchoative", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "unsafe_code": false, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": null, "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "", "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "doc_to_target": 0, "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0, "pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/", "dtype": "float", "config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/inchoative.yaml" } }, "blimp_intransitive": { "task": "blimp_intransitive", "dataset_path": "nyu-mll/blimp", "dataset_name": "intransitive", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "unsafe_code": false, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": null, "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "", "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "doc_to_target": 0, "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0, "pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/", "dtype": "float", "config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/intransitive.yaml" } }, "blimp_irregular_past_participle_adjectives": { "task": "blimp_irregular_past_participle_adjectives", "dataset_path": "nyu-mll/blimp", "dataset_name": "irregular_past_participle_adjectives", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "unsafe_code": false, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": null, "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "", "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "doc_to_target": 0, "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0, "pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/", "dtype": "float", "config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/irregular_past_participle_adjectives.yaml" } }, "blimp_irregular_past_participle_verbs": { "task": "blimp_irregular_past_participle_verbs", "dataset_path": "nyu-mll/blimp", "dataset_name": "irregular_past_participle_verbs", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "unsafe_code": false, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": null, "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "", "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "doc_to_target": 0, "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0, "pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/", "dtype": "float", "config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/irregular_past_participle_verbs.yaml" } }, "blimp_irregular_plural_subject_verb_agreement_1": { "task": "blimp_irregular_plural_subject_verb_agreement_1", "dataset_path": "nyu-mll/blimp", "dataset_name": "irregular_plural_subject_verb_agreement_1", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "unsafe_code": false, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": null, "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "", "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "doc_to_target": 0, "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0, "pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/", "dtype": "float", "config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_1.yaml" } }, "blimp_irregular_plural_subject_verb_agreement_2": { "task": "blimp_irregular_plural_subject_verb_agreement_2", "dataset_path": "nyu-mll/blimp", "dataset_name": "irregular_plural_subject_verb_agreement_2", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "unsafe_code": false, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": null, "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "", "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "doc_to_target": 0, "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0, "pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/", "dtype": "float", "config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_2.yaml" } }, "blimp_left_branch_island_echo_question": { "task": "blimp_left_branch_island_echo_question", "dataset_path": "nyu-mll/blimp", "dataset_name": "left_branch_island_echo_question", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "unsafe_code": false, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": null, "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "", "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "doc_to_target": 0, "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0, "pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/", "dtype": "float", "config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/left_branch_island_echo_question.yaml" } }, "blimp_left_branch_island_simple_question": { "task": "blimp_left_branch_island_simple_question", "dataset_path": "nyu-mll/blimp", "dataset_name": "left_branch_island_simple_question", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "unsafe_code": false, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": null, "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "", "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "doc_to_target": 0, "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0, "pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/", "dtype": "float", "config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/left_branch_island_simple_question.yaml" } }, "blimp_matrix_question_npi_licensor_present": { "task": "blimp_matrix_question_npi_licensor_present", "dataset_path": "nyu-mll/blimp", "dataset_name": "matrix_question_npi_licensor_present", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "unsafe_code": false, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": null, "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "", "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "doc_to_target": 0, "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0, "pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/", "dtype": "float", "config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/matrix_question_npi_licensor_present.yaml" } }, "blimp_npi_present_1": { "task": "blimp_npi_present_1", "dataset_path": "nyu-mll/blimp", "dataset_name": "npi_present_1", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "unsafe_code": false, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": null, "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "", "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "doc_to_target": 0, "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0, "pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/", "dtype": "float", "config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/npi_present_1.yaml" } }, "blimp_npi_present_2": { "task": "blimp_npi_present_2", "dataset_path": "nyu-mll/blimp", "dataset_name": "npi_present_2", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "unsafe_code": false, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": null, "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "", "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "doc_to_target": 0, "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0, "pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/", "dtype": "float", "config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/npi_present_2.yaml" } }, "blimp_only_npi_licensor_present": { "task": "blimp_only_npi_licensor_present", "dataset_path": "nyu-mll/blimp", "dataset_name": "only_npi_licensor_present", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "unsafe_code": false, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": null, "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "", "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "doc_to_target": 0, "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0, "pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/", "dtype": "float", "config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/only_npi_licensor_present.yaml" } }, "blimp_only_npi_scope": { "task": "blimp_only_npi_scope", "dataset_path": "nyu-mll/blimp", "dataset_name": "only_npi_scope", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "unsafe_code": false, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": null, "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "", "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "doc_to_target": 0, "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0, "pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/", "dtype": "float", "config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/only_npi_scope.yaml" } }, "blimp_passive_1": { "task": "blimp_passive_1", "dataset_path": "nyu-mll/blimp", "dataset_name": "passive_1", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "unsafe_code": false, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": null, "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "", "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "doc_to_target": 0, "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0, "pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/", "dtype": "float", "config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/passive_1.yaml" } }, "blimp_passive_2": { "task": "blimp_passive_2", "dataset_path": "nyu-mll/blimp", "dataset_name": "passive_2", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "unsafe_code": false, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": null, "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "", "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "doc_to_target": 0, "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0, "pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/", "dtype": "float", "config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/passive_2.yaml" } }, "blimp_principle_A_c_command": { "task": "blimp_principle_A_c_command", "dataset_path": "nyu-mll/blimp", "dataset_name": "principle_A_c_command", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "unsafe_code": false, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": null, "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "", "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "doc_to_target": 0, "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0, "pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/", "dtype": "float", "config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/principle_A_c_command.yaml" } }, "blimp_principle_A_case_1": { "task": "blimp_principle_A_case_1", "dataset_path": "nyu-mll/blimp", "dataset_name": "principle_A_case_1", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "unsafe_code": false, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": null, "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "", "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "doc_to_target": 0, "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0, "pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/", "dtype": "float", "config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/principle_A_case_1.yaml" } }, "blimp_principle_A_case_2": { "task": "blimp_principle_A_case_2", "dataset_path": "nyu-mll/blimp", "dataset_name": "principle_A_case_2", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "unsafe_code": false, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": null, "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "", "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "doc_to_target": 0, "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0, "pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/", "dtype": "float", "config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/principle_A_case_2.yaml" } }, "blimp_principle_A_domain_1": { "task": "blimp_principle_A_domain_1", "dataset_path": "nyu-mll/blimp", "dataset_name": "principle_A_domain_1", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "unsafe_code": false, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": null, "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "", "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "doc_to_target": 0, "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0, "pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/", "dtype": "float", "config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/principle_A_domain_1.yaml" } }, "blimp_principle_A_domain_2": { "task": "blimp_principle_A_domain_2", "dataset_path": "nyu-mll/blimp", "dataset_name": "principle_A_domain_2", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "unsafe_code": false, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": null, "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "", "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "doc_to_target": 0, "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0, "pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/", "dtype": "float", "config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/principle_A_domain_2.yaml" } }, "blimp_principle_A_domain_3": { "task": "blimp_principle_A_domain_3", "dataset_path": "nyu-mll/blimp", "dataset_name": "principle_A_domain_3", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "unsafe_code": false, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": null, "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "", "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "doc_to_target": 0, "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0, "pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/", "dtype": "float", "config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/principle_A_domain_3.yaml" } }, "blimp_principle_A_reconstruction": { "task": "blimp_principle_A_reconstruction", "dataset_path": "nyu-mll/blimp", "dataset_name": "principle_A_reconstruction", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "unsafe_code": false, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": null, "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "", "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "doc_to_target": 0, "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0, "pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/", "dtype": "float", "config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/principle_A_reconstruction.yaml" } }, "blimp_regular_plural_subject_verb_agreement_1": { "task": "blimp_regular_plural_subject_verb_agreement_1", "dataset_path": "nyu-mll/blimp", "dataset_name": "regular_plural_subject_verb_agreement_1", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "unsafe_code": false, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": null, "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "", "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "doc_to_target": 0, "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0, "pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/", "dtype": "float", "config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_1.yaml" } }, "blimp_regular_plural_subject_verb_agreement_2": { "task": "blimp_regular_plural_subject_verb_agreement_2", "dataset_path": "nyu-mll/blimp", "dataset_name": "regular_plural_subject_verb_agreement_2", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "unsafe_code": false, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": null, "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "", "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "doc_to_target": 0, "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0, "pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/", "dtype": "float", "config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_2.yaml" } }, "blimp_sentential_negation_npi_licensor_present": { "task": "blimp_sentential_negation_npi_licensor_present", "dataset_path": "nyu-mll/blimp", "dataset_name": "sentential_negation_npi_licensor_present", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "unsafe_code": false, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": null, "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "", "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "doc_to_target": 0, "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0, "pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/", "dtype": "float", "config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/sentential_negation_npi_licensor_present.yaml" } }, "blimp_sentential_negation_npi_scope": { "task": "blimp_sentential_negation_npi_scope", "dataset_path": "nyu-mll/blimp", "dataset_name": "sentential_negation_npi_scope", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "unsafe_code": false, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": null, "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "", "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "doc_to_target": 0, "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0, "pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/", "dtype": "float", "config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/sentential_negation_npi_scope.yaml" } }, "blimp_sentential_subject_island": { "task": "blimp_sentential_subject_island", "dataset_path": "nyu-mll/blimp", "dataset_name": "sentential_subject_island", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "unsafe_code": false, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": null, "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "", "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "doc_to_target": 0, "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0, "pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/", "dtype": "float", "config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/sentential_subject_island.yaml" } }, "blimp_superlative_quantifiers_1": { "task": "blimp_superlative_quantifiers_1", "dataset_path": "nyu-mll/blimp", "dataset_name": "superlative_quantifiers_1", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "unsafe_code": false, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": null, "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "", "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "doc_to_target": 0, "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0, "pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/", "dtype": "float", "config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/superlative_quantifiers_1.yaml" } }, "blimp_superlative_quantifiers_2": { "task": "blimp_superlative_quantifiers_2", "dataset_path": "nyu-mll/blimp", "dataset_name": "superlative_quantifiers_2", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "unsafe_code": false, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": null, "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "", "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "doc_to_target": 0, "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0, "pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/", "dtype": "float", "config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/superlative_quantifiers_2.yaml" } }, "blimp_tough_vs_raising_1": { "task": "blimp_tough_vs_raising_1", "dataset_path": "nyu-mll/blimp", "dataset_name": "tough_vs_raising_1", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "unsafe_code": false, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": null, "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "", "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "doc_to_target": 0, "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0, "pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/", "dtype": "float", "config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/tough_vs_raising_1.yaml" } }, "blimp_tough_vs_raising_2": { "task": "blimp_tough_vs_raising_2", "dataset_path": "nyu-mll/blimp", "dataset_name": "tough_vs_raising_2", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "unsafe_code": false, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": null, "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "", "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "doc_to_target": 0, "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0, "pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/", "dtype": "float", "config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/tough_vs_raising_2.yaml" } }, "blimp_transitive": { "task": "blimp_transitive", "dataset_path": "nyu-mll/blimp", "dataset_name": "transitive", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "unsafe_code": false, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": null, "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "", "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "doc_to_target": 0, "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0, "pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/", "dtype": "float", "config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/transitive.yaml" } }, "blimp_wh_island": { "task": "blimp_wh_island", "dataset_path": "nyu-mll/blimp", "dataset_name": "wh_island", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "unsafe_code": false, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": null, "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "", "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "doc_to_target": 0, "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0, "pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/", "dtype": "float", "config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/wh_island.yaml" } }, "blimp_wh_questions_object_gap": { "task": "blimp_wh_questions_object_gap", "dataset_path": "nyu-mll/blimp", "dataset_name": "wh_questions_object_gap", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "unsafe_code": false, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": null, "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "", "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "doc_to_target": 0, "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0, "pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/", "dtype": "float", "config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/wh_questions_object_gap.yaml" } }, "blimp_wh_questions_subject_gap": { "task": "blimp_wh_questions_subject_gap", "dataset_path": "nyu-mll/blimp", "dataset_name": "wh_questions_subject_gap", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "unsafe_code": false, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": null, "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "", "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "doc_to_target": 0, "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0, "pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/", "dtype": "float", "config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/wh_questions_subject_gap.yaml" } }, "blimp_wh_questions_subject_gap_long_distance": { "task": "blimp_wh_questions_subject_gap_long_distance", "dataset_path": "nyu-mll/blimp", "dataset_name": "wh_questions_subject_gap_long_distance", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "unsafe_code": false, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": null, "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "", "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "doc_to_target": 0, "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0, "pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/", "dtype": "float", "config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/wh_questions_subject_gap_long_distance.yaml" } }, "blimp_wh_vs_that_no_gap": { "task": "blimp_wh_vs_that_no_gap", "dataset_path": "nyu-mll/blimp", "dataset_name": "wh_vs_that_no_gap", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "unsafe_code": false, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": null, "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "", "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "doc_to_target": 0, "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0, "pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/", "dtype": "float", "config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/wh_vs_that_no_gap.yaml" } }, "blimp_wh_vs_that_no_gap_long_distance": { "task": "blimp_wh_vs_that_no_gap_long_distance", "dataset_path": "nyu-mll/blimp", "dataset_name": "wh_vs_that_no_gap_long_distance", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "unsafe_code": false, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": null, "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "", "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "doc_to_target": 0, "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0, "pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/", "dtype": "float", "config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/wh_vs_that_no_gap_long_distance.yaml" } }, "blimp_wh_vs_that_with_gap": { "task": "blimp_wh_vs_that_with_gap", "dataset_path": "nyu-mll/blimp", "dataset_name": "wh_vs_that_with_gap", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "unsafe_code": false, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": null, "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "", "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "doc_to_target": 0, "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0, "pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/", "dtype": "float", "config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/wh_vs_that_with_gap.yaml" } }, "blimp_wh_vs_that_with_gap_long_distance": { "task": "blimp_wh_vs_that_with_gap_long_distance", "dataset_path": "nyu-mll/blimp", "dataset_name": "wh_vs_that_with_gap_long_distance", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "unsafe_code": false, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": null, "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "", "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "doc_to_target": 0, "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0, "pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/", "dtype": "float", "config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/blimp/wh_vs_that_with_gap_long_distance.yaml" } }, "boolq": { "task": "boolq", "dataset_path": "aps/super_glue", "dataset_name": "boolq", "training_split": "train", "validation_split": "validation", "doc_to_text": "{{passage}}\nQuestion: {{question}}?\nAnswer:", "doc_to_target": "label", "unsafe_code": false, "doc_to_choice": [ "no", "yes" ], "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": null, "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "{{passage}}\nQuestion: {{question}}?\nAnswer:", "doc_to_choice": [ "no", "yes" ], "doc_to_target": "label", "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc" } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "passage", "metadata": { "version": 2.0, "pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/", "dtype": "float", "config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/super_glue/boolq/default.yaml" } }, "hellaswag": { "task": "hellaswag", "dataset_path": "Rowan/hellaswag", "training_split": "train", "validation_split": "validation", "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n out_doc = {\n \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n \"gold\": int(doc[\"label\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", "doc_to_text": "{{query}}", "doc_to_target": "{{label}}", "unsafe_code": false, "doc_to_choice": "choices", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": null, "process_docs": "", "fewshot_indices": null, "samples": null, "doc_to_text": "{{query}}", "doc_to_choice": "choices", "doc_to_target": "{{label}}", "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true }, { "metric": "acc_norm", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0, "pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/", "dtype": "float", "config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/hellaswag/hellaswag.yaml" } }, "piqa": { "task": "piqa", "dataset_path": "baber/piqa", "training_split": "train", "validation_split": "validation", "doc_to_text": "Question: {{goal}}\nAnswer:", "doc_to_target": "label", "unsafe_code": false, "doc_to_choice": "{{[sol1, sol2]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": null, "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "Question: {{goal}}\nAnswer:", "doc_to_choice": "{{[sol1, sol2]}}", "doc_to_target": "label", "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true }, { "metric": "acc_norm", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "goal", "metadata": { "version": 1.0, "pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/", "dtype": "float", "config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/piqa/piqa.yaml" } }, "wikitext": { "task": "wikitext", "dataset_path": "EleutherAI/wikitext_document_level", "dataset_name": "wikitext-2-raw-v1", "training_split": "train", "validation_split": "validation", "test_split": "test", "doc_to_text": "", "doc_to_target": "def wikitext_detokenizer(doc):\n string = doc[\"page\"]\n # contractions\n string = string.replace(\"s '\", \"s'\")\n string = re.sub(r\"/' [0-9]/\", r\"/'[0-9]/\", string)\n # number separators\n string = string.replace(\" @-@ \", \"-\")\n string = string.replace(\" @,@ \", \",\")\n string = string.replace(\" @.@ \", \".\")\n # punctuation\n string = string.replace(\" : \", \": \")\n string = string.replace(\" ; \", \"; \")\n string = string.replace(\" . \", \". \")\n string = string.replace(\" ! \", \"! \")\n string = string.replace(\" ? \", \"? \")\n string = string.replace(\" , \", \", \")\n # double brackets\n string = re.sub(r\"\\(\\s*([^\\)]*?)\\s*\\)\", r\"(\\1)\", string)\n string = re.sub(r\"\\[\\s*([^\\]]*?)\\s*\\]\", r\"[\\1]\", string)\n string = re.sub(r\"{\\s*([^}]*?)\\s*}\", r\"{\\1}\", string)\n string = re.sub(r\"\\\"\\s*([^\\\"]*?)\\s*\\\"\", r'\"\\1\"', string)\n string = re.sub(r\"'\\s*([^']*?)\\s*'\", r\"'\\1'\", string)\n # miscellaneous\n string = string.replace(\"= = = =\", \"====\")\n string = string.replace(\"= = =\", \"===\")\n string = string.replace(\"= =\", \"==\")\n string = string.replace(\" \" + chr(176) + \" \", chr(176))\n string = string.replace(\" \\n\", \"\\n\")\n string = string.replace(\"\\n \", \"\\n\")\n string = string.replace(\" N \", \" 1 \")\n string = string.replace(\" 's\", \"'s\")\n\n return string\n", "unsafe_code": false, "process_results": "def process_results(doc, results):\n (loglikelihood,) = results\n # IMPORTANT: wikitext counts number of words in *original doc before detokenization*\n _words = len(re.split(r\"\\s+\", doc[\"page\"]))\n _bytes = len(doc[\"page\"].encode(\"utf-8\"))\n return {\n \"word_perplexity\": (loglikelihood, _words),\n \"byte_perplexity\": (loglikelihood, _bytes),\n \"bits_per_byte\": (loglikelihood, _bytes),\n }\n", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": null, "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "", "doc_to_choice": null, "doc_to_target": "", "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "word_perplexity" }, { "metric": "byte_perplexity" }, { "metric": "bits_per_byte" } ], "output_type": "loglikelihood_rolling", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{page}}", "metadata": { "version": 2.0, "pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/", "dtype": "float", "config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/wikitext/wikitext.yaml" } }, "winogrande": { "task": "winogrande", "dataset_path": "allenai/winogrande", "dataset_name": "winogrande_xl", "training_split": "train", "validation_split": "validation", "doc_to_text": "def doc_to_text(doc):\n answer_to_num = {\"1\": 0, \"2\": 1}\n return answer_to_num[doc[\"answer\"]]\n", "doc_to_target": "def doc_to_target(doc):\n idx = doc[\"sentence\"].index(\"_\") + 1\n return doc[\"sentence\"][idx:].strip()\n", "unsafe_code": false, "doc_to_choice": "def doc_to_choice(doc):\n idx = doc[\"sentence\"].index(\"_\")\n options = [doc[\"option1\"], doc[\"option2\"]]\n return [doc[\"sentence\"][:idx] + opt for opt in options]\n", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": null, "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "", "doc_to_choice": "", "doc_to_target": "", "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "sentence", "metadata": { "version": 1.0, "pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/", "dtype": "float", "config_source": "/home/nvidia/lm-evaluation-harness/lm_eval/tasks/winogrande/default.yaml" } } }, "versions": { "arc_challenge": 1.0, "arc_easy": 1.0, "blimp": "2.0", "blimp_adjunct_island": 1.0, "blimp_anaphor_gender_agreement": 1.0, "blimp_anaphor_number_agreement": 1.0, "blimp_animate_subject_passive": 1.0, "blimp_animate_subject_trans": 1.0, "blimp_causative": 1.0, "blimp_complex_NP_island": 1.0, "blimp_coordinate_structure_constraint_complex_left_branch": 1.0, "blimp_coordinate_structure_constraint_object_extraction": 1.0, "blimp_determiner_noun_agreement_1": 1.0, "blimp_determiner_noun_agreement_2": 1.0, "blimp_determiner_noun_agreement_irregular_1": 1.0, "blimp_determiner_noun_agreement_irregular_2": 1.0, "blimp_determiner_noun_agreement_with_adj_2": 1.0, "blimp_determiner_noun_agreement_with_adj_irregular_1": 1.0, "blimp_determiner_noun_agreement_with_adj_irregular_2": 1.0, "blimp_determiner_noun_agreement_with_adjective_1": 1.0, "blimp_distractor_agreement_relational_noun": 1.0, "blimp_distractor_agreement_relative_clause": 1.0, "blimp_drop_argument": 1.0, "blimp_ellipsis_n_bar_1": 1.0, "blimp_ellipsis_n_bar_2": 1.0, "blimp_existential_there_object_raising": 1.0, "blimp_existential_there_quantifiers_1": 1.0, "blimp_existential_there_quantifiers_2": 1.0, "blimp_existential_there_subject_raising": 1.0, "blimp_expletive_it_object_raising": 1.0, "blimp_inchoative": 1.0, "blimp_intransitive": 1.0, "blimp_irregular_past_participle_adjectives": 1.0, "blimp_irregular_past_participle_verbs": 1.0, "blimp_irregular_plural_subject_verb_agreement_1": 1.0, "blimp_irregular_plural_subject_verb_agreement_2": 1.0, "blimp_left_branch_island_echo_question": 1.0, "blimp_left_branch_island_simple_question": 1.0, "blimp_matrix_question_npi_licensor_present": 1.0, "blimp_npi_present_1": 1.0, "blimp_npi_present_2": 1.0, "blimp_only_npi_licensor_present": 1.0, "blimp_only_npi_scope": 1.0, "blimp_passive_1": 1.0, "blimp_passive_2": 1.0, "blimp_principle_A_c_command": 1.0, "blimp_principle_A_case_1": 1.0, "blimp_principle_A_case_2": 1.0, "blimp_principle_A_domain_1": 1.0, "blimp_principle_A_domain_2": 1.0, "blimp_principle_A_domain_3": 1.0, "blimp_principle_A_reconstruction": 1.0, "blimp_regular_plural_subject_verb_agreement_1": 1.0, "blimp_regular_plural_subject_verb_agreement_2": 1.0, "blimp_sentential_negation_npi_licensor_present": 1.0, "blimp_sentential_negation_npi_scope": 1.0, "blimp_sentential_subject_island": 1.0, "blimp_superlative_quantifiers_1": 1.0, "blimp_superlative_quantifiers_2": 1.0, "blimp_tough_vs_raising_1": 1.0, "blimp_tough_vs_raising_2": 1.0, "blimp_transitive": 1.0, "blimp_wh_island": 1.0, "blimp_wh_questions_object_gap": 1.0, "blimp_wh_questions_subject_gap": 1.0, "blimp_wh_questions_subject_gap_long_distance": 1.0, "blimp_wh_vs_that_no_gap": 1.0, "blimp_wh_vs_that_no_gap_long_distance": 1.0, "blimp_wh_vs_that_with_gap": 1.0, "blimp_wh_vs_that_with_gap_long_distance": 1.0, "boolq": 2.0, "hellaswag": 1.0, "piqa": 1.0, "wikitext": 2.0, "winogrande": 1.0 }, "n-shot": { "arc_challenge": 0, "arc_easy": 0, "blimp": 0, "blimp_adjunct_island": 0, "blimp_anaphor_gender_agreement": 0, "blimp_anaphor_number_agreement": 0, "blimp_animate_subject_passive": 0, "blimp_animate_subject_trans": 0, "blimp_causative": 0, "blimp_complex_NP_island": 0, "blimp_coordinate_structure_constraint_complex_left_branch": 0, "blimp_coordinate_structure_constraint_object_extraction": 0, "blimp_determiner_noun_agreement_1": 0, "blimp_determiner_noun_agreement_2": 0, "blimp_determiner_noun_agreement_irregular_1": 0, "blimp_determiner_noun_agreement_irregular_2": 0, "blimp_determiner_noun_agreement_with_adj_2": 0, "blimp_determiner_noun_agreement_with_adj_irregular_1": 0, "blimp_determiner_noun_agreement_with_adj_irregular_2": 0, "blimp_determiner_noun_agreement_with_adjective_1": 0, "blimp_distractor_agreement_relational_noun": 0, "blimp_distractor_agreement_relative_clause": 0, "blimp_drop_argument": 0, "blimp_ellipsis_n_bar_1": 0, "blimp_ellipsis_n_bar_2": 0, "blimp_existential_there_object_raising": 0, "blimp_existential_there_quantifiers_1": 0, "blimp_existential_there_quantifiers_2": 0, "blimp_existential_there_subject_raising": 0, "blimp_expletive_it_object_raising": 0, "blimp_inchoative": 0, "blimp_intransitive": 0, "blimp_irregular_past_participle_adjectives": 0, "blimp_irregular_past_participle_verbs": 0, "blimp_irregular_plural_subject_verb_agreement_1": 0, "blimp_irregular_plural_subject_verb_agreement_2": 0, "blimp_left_branch_island_echo_question": 0, "blimp_left_branch_island_simple_question": 0, "blimp_matrix_question_npi_licensor_present": 0, "blimp_npi_present_1": 0, "blimp_npi_present_2": 0, "blimp_only_npi_licensor_present": 0, "blimp_only_npi_scope": 0, "blimp_passive_1": 0, "blimp_passive_2": 0, "blimp_principle_A_c_command": 0, "blimp_principle_A_case_1": 0, "blimp_principle_A_case_2": 0, "blimp_principle_A_domain_1": 0, "blimp_principle_A_domain_2": 0, "blimp_principle_A_domain_3": 0, "blimp_principle_A_reconstruction": 0, "blimp_regular_plural_subject_verb_agreement_1": 0, "blimp_regular_plural_subject_verb_agreement_2": 0, "blimp_sentential_negation_npi_licensor_present": 0, "blimp_sentential_negation_npi_scope": 0, "blimp_sentential_subject_island": 0, "blimp_superlative_quantifiers_1": 0, "blimp_superlative_quantifiers_2": 0, "blimp_tough_vs_raising_1": 0, "blimp_tough_vs_raising_2": 0, "blimp_transitive": 0, "blimp_wh_island": 0, "blimp_wh_questions_object_gap": 0, "blimp_wh_questions_subject_gap": 0, "blimp_wh_questions_subject_gap_long_distance": 0, "blimp_wh_vs_that_no_gap": 0, "blimp_wh_vs_that_no_gap_long_distance": 0, "blimp_wh_vs_that_with_gap": 0, "blimp_wh_vs_that_with_gap_long_distance": 0, "boolq": 0, "hellaswag": 0, "piqa": 0, "wikitext": 0, "winogrande": 0 }, "higher_is_better": { "arc_challenge": { "acc": true, "acc_norm": true }, "arc_easy": { "acc": true, "acc_norm": true }, "blimp": { "acc": true }, "blimp_adjunct_island": { "acc": true }, "blimp_anaphor_gender_agreement": { "acc": true }, "blimp_anaphor_number_agreement": { "acc": true }, "blimp_animate_subject_passive": { "acc": true }, "blimp_animate_subject_trans": { "acc": true }, "blimp_causative": { "acc": true }, "blimp_complex_NP_island": { "acc": true }, "blimp_coordinate_structure_constraint_complex_left_branch": { "acc": true }, "blimp_coordinate_structure_constraint_object_extraction": { "acc": true }, "blimp_determiner_noun_agreement_1": { "acc": true }, "blimp_determiner_noun_agreement_2": { "acc": true }, "blimp_determiner_noun_agreement_irregular_1": { "acc": true }, "blimp_determiner_noun_agreement_irregular_2": { "acc": true }, "blimp_determiner_noun_agreement_with_adj_2": { "acc": true }, "blimp_determiner_noun_agreement_with_adj_irregular_1": { "acc": true }, "blimp_determiner_noun_agreement_with_adj_irregular_2": { "acc": true }, "blimp_determiner_noun_agreement_with_adjective_1": { "acc": true }, "blimp_distractor_agreement_relational_noun": { "acc": true }, "blimp_distractor_agreement_relative_clause": { "acc": true }, "blimp_drop_argument": { "acc": true }, "blimp_ellipsis_n_bar_1": { "acc": true }, "blimp_ellipsis_n_bar_2": { "acc": true }, "blimp_existential_there_object_raising": { "acc": true }, "blimp_existential_there_quantifiers_1": { "acc": true }, "blimp_existential_there_quantifiers_2": { "acc": true }, "blimp_existential_there_subject_raising": { "acc": true }, "blimp_expletive_it_object_raising": { "acc": true }, "blimp_inchoative": { "acc": true }, "blimp_intransitive": { "acc": true }, "blimp_irregular_past_participle_adjectives": { "acc": true }, "blimp_irregular_past_participle_verbs": { "acc": true }, "blimp_irregular_plural_subject_verb_agreement_1": { "acc": true }, "blimp_irregular_plural_subject_verb_agreement_2": { "acc": true }, "blimp_left_branch_island_echo_question": { "acc": true }, "blimp_left_branch_island_simple_question": { "acc": true }, "blimp_matrix_question_npi_licensor_present": { "acc": true }, "blimp_npi_present_1": { "acc": true }, "blimp_npi_present_2": { "acc": true }, "blimp_only_npi_licensor_present": { "acc": true }, "blimp_only_npi_scope": { "acc": true }, "blimp_passive_1": { "acc": true }, "blimp_passive_2": { "acc": true }, "blimp_principle_A_c_command": { "acc": true }, "blimp_principle_A_case_1": { "acc": true }, "blimp_principle_A_case_2": { "acc": true }, "blimp_principle_A_domain_1": { "acc": true }, "blimp_principle_A_domain_2": { "acc": true }, "blimp_principle_A_domain_3": { "acc": true }, "blimp_principle_A_reconstruction": { "acc": true }, "blimp_regular_plural_subject_verb_agreement_1": { "acc": true }, "blimp_regular_plural_subject_verb_agreement_2": { "acc": true }, "blimp_sentential_negation_npi_licensor_present": { "acc": true }, "blimp_sentential_negation_npi_scope": { "acc": true }, "blimp_sentential_subject_island": { "acc": true }, "blimp_superlative_quantifiers_1": { "acc": true }, "blimp_superlative_quantifiers_2": { "acc": true }, "blimp_tough_vs_raising_1": { "acc": true }, "blimp_tough_vs_raising_2": { "acc": true }, "blimp_transitive": { "acc": true }, "blimp_wh_island": { "acc": true }, "blimp_wh_questions_object_gap": { "acc": true }, "blimp_wh_questions_subject_gap": { "acc": true }, "blimp_wh_questions_subject_gap_long_distance": { "acc": true }, "blimp_wh_vs_that_no_gap": { "acc": true }, "blimp_wh_vs_that_no_gap_long_distance": { "acc": true }, "blimp_wh_vs_that_with_gap": { "acc": true }, "blimp_wh_vs_that_with_gap_long_distance": { "acc": true }, "boolq": { "acc": true }, "hellaswag": { "acc": true, "acc_norm": true }, "piqa": { "acc": true, "acc_norm": true }, "wikitext": { "word_perplexity": false, "byte_perplexity": false, "bits_per_byte": false }, "winogrande": { "acc": true } }, "n-samples": { "winogrande": { "original": 1267, "effective": 1267 }, "arc_easy": { "original": 2376, "effective": 2376 }, "hellaswag": { "original": 10042, "effective": 10042 }, "boolq": { "original": 3270, "effective": 3270 }, "arc_challenge": { "original": 1172, "effective": 1172 }, "piqa": { "original": 1838, "effective": 1838 }, "blimp_adjunct_island": { "original": 1000, "effective": 1000 }, "blimp_anaphor_gender_agreement": { "original": 1000, "effective": 1000 }, "blimp_anaphor_number_agreement": { "original": 1000, "effective": 1000 }, "blimp_animate_subject_passive": { "original": 1000, "effective": 1000 }, "blimp_animate_subject_trans": { "original": 1000, "effective": 1000 }, "blimp_causative": { "original": 1000, "effective": 1000 }, "blimp_complex_NP_island": { "original": 1000, "effective": 1000 }, "blimp_coordinate_structure_constraint_complex_left_branch": { "original": 1000, "effective": 1000 }, "blimp_coordinate_structure_constraint_object_extraction": { "original": 1000, "effective": 1000 }, "blimp_determiner_noun_agreement_1": { "original": 1000, "effective": 1000 }, "blimp_determiner_noun_agreement_2": { "original": 1000, "effective": 1000 }, "blimp_determiner_noun_agreement_irregular_1": { "original": 1000, "effective": 1000 }, "blimp_determiner_noun_agreement_irregular_2": { "original": 1000, "effective": 1000 }, "blimp_determiner_noun_agreement_with_adj_2": { "original": 1000, "effective": 1000 }, "blimp_determiner_noun_agreement_with_adj_irregular_1": { "original": 1000, "effective": 1000 }, "blimp_determiner_noun_agreement_with_adj_irregular_2": { "original": 1000, "effective": 1000 }, "blimp_determiner_noun_agreement_with_adjective_1": { "original": 1000, "effective": 1000 }, "blimp_distractor_agreement_relational_noun": { "original": 1000, "effective": 1000 }, "blimp_distractor_agreement_relative_clause": { "original": 1000, "effective": 1000 }, "blimp_drop_argument": { "original": 1000, "effective": 1000 }, "blimp_ellipsis_n_bar_1": { "original": 1000, "effective": 1000 }, "blimp_ellipsis_n_bar_2": { "original": 1000, "effective": 1000 }, "blimp_existential_there_object_raising": { "original": 1000, "effective": 1000 }, "blimp_existential_there_quantifiers_1": { "original": 1000, "effective": 1000 }, "blimp_existential_there_quantifiers_2": { "original": 1000, "effective": 1000 }, "blimp_existential_there_subject_raising": { "original": 1000, "effective": 1000 }, "blimp_expletive_it_object_raising": { "original": 1000, "effective": 1000 }, "blimp_inchoative": { "original": 1000, "effective": 1000 }, "blimp_intransitive": { "original": 1000, "effective": 1000 }, "blimp_irregular_past_participle_adjectives": { "original": 1000, "effective": 1000 }, "blimp_irregular_past_participle_verbs": { "original": 1000, "effective": 1000 }, "blimp_irregular_plural_subject_verb_agreement_1": { "original": 1000, "effective": 1000 }, "blimp_irregular_plural_subject_verb_agreement_2": { "original": 1000, "effective": 1000 }, "blimp_left_branch_island_echo_question": { "original": 1000, "effective": 1000 }, "blimp_left_branch_island_simple_question": { "original": 1000, "effective": 1000 }, "blimp_matrix_question_npi_licensor_present": { "original": 1000, "effective": 1000 }, "blimp_npi_present_1": { "original": 1000, "effective": 1000 }, "blimp_npi_present_2": { "original": 1000, "effective": 1000 }, "blimp_only_npi_licensor_present": { "original": 1000, "effective": 1000 }, "blimp_only_npi_scope": { "original": 1000, "effective": 1000 }, "blimp_passive_1": { "original": 1000, "effective": 1000 }, "blimp_passive_2": { "original": 1000, "effective": 1000 }, "blimp_principle_A_c_command": { "original": 1000, "effective": 1000 }, "blimp_principle_A_case_1": { "original": 1000, "effective": 1000 }, "blimp_principle_A_case_2": { "original": 1000, "effective": 1000 }, "blimp_principle_A_domain_1": { "original": 1000, "effective": 1000 }, "blimp_principle_A_domain_2": { "original": 1000, "effective": 1000 }, "blimp_principle_A_domain_3": { "original": 1000, "effective": 1000 }, "blimp_principle_A_reconstruction": { "original": 1000, "effective": 1000 }, "blimp_regular_plural_subject_verb_agreement_1": { "original": 1000, "effective": 1000 }, "blimp_regular_plural_subject_verb_agreement_2": { "original": 1000, "effective": 1000 }, "blimp_sentential_negation_npi_licensor_present": { "original": 1000, "effective": 1000 }, "blimp_sentential_negation_npi_scope": { "original": 1000, "effective": 1000 }, "blimp_sentential_subject_island": { "original": 1000, "effective": 1000 }, "blimp_superlative_quantifiers_1": { "original": 1000, "effective": 1000 }, "blimp_superlative_quantifiers_2": { "original": 1000, "effective": 1000 }, "blimp_tough_vs_raising_1": { "original": 1000, "effective": 1000 }, "blimp_tough_vs_raising_2": { "original": 1000, "effective": 1000 }, "blimp_transitive": { "original": 1000, "effective": 1000 }, "blimp_wh_island": { "original": 1000, "effective": 1000 }, "blimp_wh_questions_object_gap": { "original": 1000, "effective": 1000 }, "blimp_wh_questions_subject_gap": { "original": 1000, "effective": 1000 }, "blimp_wh_questions_subject_gap_long_distance": { "original": 1000, "effective": 1000 }, "blimp_wh_vs_that_no_gap": { "original": 1000, "effective": 1000 }, "blimp_wh_vs_that_no_gap_long_distance": { "original": 1000, "effective": 1000 }, "blimp_wh_vs_that_with_gap": { "original": 1000, "effective": 1000 }, "blimp_wh_vs_that_with_gap_long_distance": { "original": 1000, "effective": 1000 }, "wikitext": { "original": 62, "effective": 62 } }, "config": { "model": "hf", "model_args": { "pretrained": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/", "dtype": "float" }, "model_num_parameters": 28355072, "model_dtype": "torch.float32", "model_revision": "main", "model_sha": "", "batch_size": "auto:4", "batch_sizes": [ 40, 64, 64, 64, 64 ], "device": "cuda1", "use_cache": null, "limit": null, "bootstrap_iters": 100000, "gen_kwargs": {}, "random_seed": 0, "numpy_seed": 1234, "torch_seed": 1234, "fewshot_seed": 1234 }, "git_hash": "1dd93108", "date": 1781007562.4607315, "pretty_env_info": "PyTorch version: 2.12.0+cu130\nIs debug build: False\nCUDA used to build PyTorch: 13.0\nROCM used to build PyTorch: N/A\n\nOS: Linux Mint 22.1 (x86_64)\nGCC version: (Ubuntu 13.3.0-6ubuntu2~24.04) 13.3.0\nClang version: Could not collect\nCMake version: version 3.28.3\nLibc version: glibc-2.39\n\nPython version: 3.12.3 (main, Mar 23 2026, 19:04:32) [GCC 13.3.0] (64-bit runtime)\nPython platform: Linux-6.8.0-51-generic-x86_64-with-glibc2.39\nIs CUDA available: True\nCUDA runtime version: 12.0.140\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA GeForce RTX 3060\nGPU 1: NVIDIA GeForce RTX 2080 Ti\n\nNvidia driver version: 595.71.05\ncuDNN version: Could not collect\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\nCaching allocator config: N/A\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 36\nOn-line CPU(s) list: 0-35\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) CPU E5-2695 v4 @ 2.10GHz\nCPU family: 6\nModel: 79\nThread(s) per core: 2\nCore(s) per socket: 18\nSocket(s): 1\nStepping: 1\nCPU(s) scaling MHz: 45%\nCPU max MHz: 3300,0000\nCPU min MHz: 1200,0000\nBogoMIPS: 4190,53\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cdp_l3 pti intel_ppin ssbd ibrs ibpb stibp tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm cqm rdt_a rdseed adx smap intel_pt xsaveopt cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local dtherm ida arat pln pts vnmi md_clear flush_l1d\nVirtualization: VT-x\nL1d cache: 576 KiB (18 instances)\nL1i cache: 576 KiB (18 instances)\nL2 cache: 4,5 MiB (18 instances)\nL3 cache: 45 MiB (1 instance)\nNUMA node(s): 1\nNUMA node0 CPU(s): 0-35\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit: KVM: Mitigation: VMX disabled\nVulnerability L1tf: Mitigation; PTE Inversion; VMX conditional cache flushes, SMT vulnerable\nVulnerability Mds: Mitigation; Clear CPU buffers; SMT vulnerable\nVulnerability Meltdown: Mitigation; PTI\nVulnerability Mmio stale data: Mitigation; Clear CPU buffers; SMT vulnerable\nVulnerability Reg file data sampling: Not affected\nVulnerability Retbleed: Not affected\nVulnerability Spec rstack overflow: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; IBPB conditional; IBRS_FW; STIBP conditional; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Mitigation; Clear CPU buffers; SMT vulnerable\n\nVersions of relevant libraries:\n[pip3] numpy==2.4.6\n[pip3] nvidia-cublas==13.1.1.3\n[pip3] nvidia-cuda-cupti==13.0.85\n[pip3] nvidia-cuda-nvrtc==13.0.88\n[pip3] nvidia-cuda-runtime==13.0.96\n[pip3] nvidia-cudnn-cu13==9.20.0.48\n[pip3] nvidia-cufft==12.0.0.61\n[pip3] nvidia-curand==10.4.0.35\n[pip3] nvidia-cusolver==12.0.4.66\n[pip3] nvidia-cusparse==12.6.3.3\n[pip3] nvidia-cusparselt-cu13==0.8.1\n[pip3] nvidia-nccl-cu13==2.29.7\n[pip3] nvidia-nvjitlink==13.0.88\n[pip3] nvidia-nvtx==13.0.85\n[pip3] torch==2.12.0\n[pip3] triton==3.7.0\n[conda] Could not collect", "transformers_version": "5.10.2", "lm_eval_version": "0.4.13.dev0", "upper_git_hash": null, "tokenizer_pad_token": [ "", "2" ], "tokenizer_eos_token": [ "<|im_end|>", "1" ], "tokenizer_bos_token": [ "", "3" ], "eot_token_id": 1, "max_length": 1024, "task_hashes": {}, "model_source": "hf", "model_name": "/home/nvidia/pretrain/michel-micro/checkpoint-80000/", "model_name_sanitized": "__home__nvidia__pretrain__michel-micro__checkpoint-80000__", "system_instruction": null, "system_instruction_sha": null, "fewshot_as_multiturn": null, "chat_template": null, "chat_template_sha": null, "total_evaluation_time_seconds": "797.8711924949894" }