421 lines
14 KiB
JSON
421 lines
14 KiB
JSON
|
{
|
||
|
|
"results": {
|
||
|
|
"hendrycksTest-abstract_algebra": {
|
||
|
|
"acc": 0.33,
|
||
|
|
"acc_stderr": 0.04725815626252605,
|
||
|
|
"acc_norm": 0.33,
|
||
|
|
"acc_norm_stderr": 0.04725815626252605
|
||
|
|
},
|
||
|
|
"hendrycksTest-anatomy": {
|
||
|
|
"acc": 0.45185185185185184,
|
||
|
|
"acc_stderr": 0.04299268905480864,
|
||
|
|
"acc_norm": 0.45185185185185184,
|
||
|
|
"acc_norm_stderr": 0.04299268905480864
|
||
|
|
},
|
||
|
|
"hendrycksTest-astronomy": {
|
||
|
|
"acc": 0.42105263157894735,
|
||
|
|
"acc_stderr": 0.040179012759817494,
|
||
|
|
"acc_norm": 0.42105263157894735,
|
||
|
|
"acc_norm_stderr": 0.040179012759817494
|
||
|
|
},
|
||
|
|
"hendrycksTest-business_ethics": {
|
||
|
|
"acc": 0.5,
|
||
|
|
"acc_stderr": 0.050251890762960605,
|
||
|
|
"acc_norm": 0.5,
|
||
|
|
"acc_norm_stderr": 0.050251890762960605
|
||
|
|
},
|
||
|
|
"hendrycksTest-clinical_knowledge": {
|
||
|
|
"acc": 0.4679245283018868,
|
||
|
|
"acc_stderr": 0.030709486992556545,
|
||
|
|
"acc_norm": 0.4679245283018868,
|
||
|
|
"acc_norm_stderr": 0.030709486992556545
|
||
|
|
},
|
||
|
|
"hendrycksTest-college_biology": {
|
||
|
|
"acc": 0.4513888888888889,
|
||
|
|
"acc_stderr": 0.04161402398403279,
|
||
|
|
"acc_norm": 0.4513888888888889,
|
||
|
|
"acc_norm_stderr": 0.04161402398403279
|
||
|
|
},
|
||
|
|
"hendrycksTest-college_chemistry": {
|
||
|
|
"acc": 0.34,
|
||
|
|
"acc_stderr": 0.04760952285695235,
|
||
|
|
"acc_norm": 0.34,
|
||
|
|
"acc_norm_stderr": 0.04760952285695235
|
||
|
|
},
|
||
|
|
"hendrycksTest-college_computer_science": {
|
||
|
|
"acc": 0.37,
|
||
|
|
"acc_stderr": 0.04852365870939099,
|
||
|
|
"acc_norm": 0.37,
|
||
|
|
"acc_norm_stderr": 0.04852365870939099
|
||
|
|
},
|
||
|
|
"hendrycksTest-college_mathematics": {
|
||
|
|
"acc": 0.35,
|
||
|
|
"acc_stderr": 0.0479372485441102,
|
||
|
|
"acc_norm": 0.35,
|
||
|
|
"acc_norm_stderr": 0.0479372485441102
|
||
|
|
},
|
||
|
|
"hendrycksTest-college_medicine": {
|
||
|
|
"acc": 0.4161849710982659,
|
||
|
|
"acc_stderr": 0.03758517775404947,
|
||
|
|
"acc_norm": 0.4161849710982659,
|
||
|
|
"acc_norm_stderr": 0.03758517775404947
|
||
|
|
},
|
||
|
|
"hendrycksTest-college_physics": {
|
||
|
|
"acc": 0.20588235294117646,
|
||
|
|
"acc_stderr": 0.04023382273617746,
|
||
|
|
"acc_norm": 0.20588235294117646,
|
||
|
|
"acc_norm_stderr": 0.04023382273617746
|
||
|
|
},
|
||
|
|
"hendrycksTest-computer_security": {
|
||
|
|
"acc": 0.56,
|
||
|
|
"acc_stderr": 0.04988876515698589,
|
||
|
|
"acc_norm": 0.56,
|
||
|
|
"acc_norm_stderr": 0.04988876515698589
|
||
|
|
},
|
||
|
|
"hendrycksTest-conceptual_physics": {
|
||
|
|
"acc": 0.37872340425531914,
|
||
|
|
"acc_stderr": 0.03170995606040655,
|
||
|
|
"acc_norm": 0.37872340425531914,
|
||
|
|
"acc_norm_stderr": 0.03170995606040655
|
||
|
|
},
|
||
|
|
"hendrycksTest-econometrics": {
|
||
|
|
"acc": 0.3333333333333333,
|
||
|
|
"acc_stderr": 0.044346007015849245,
|
||
|
|
"acc_norm": 0.3333333333333333,
|
||
|
|
"acc_norm_stderr": 0.044346007015849245
|
||
|
|
},
|
||
|
|
"hendrycksTest-electrical_engineering": {
|
||
|
|
"acc": 0.38620689655172413,
|
||
|
|
"acc_stderr": 0.04057324734419035,
|
||
|
|
"acc_norm": 0.38620689655172413,
|
||
|
|
"acc_norm_stderr": 0.04057324734419035
|
||
|
|
},
|
||
|
|
"hendrycksTest-elementary_mathematics": {
|
||
|
|
"acc": 0.2830687830687831,
|
||
|
|
"acc_stderr": 0.023201392938194974,
|
||
|
|
"acc_norm": 0.2830687830687831,
|
||
|
|
"acc_norm_stderr": 0.023201392938194974
|
||
|
|
},
|
||
|
|
"hendrycksTest-formal_logic": {
|
||
|
|
"acc": 0.2222222222222222,
|
||
|
|
"acc_stderr": 0.037184890068181146,
|
||
|
|
"acc_norm": 0.2222222222222222,
|
||
|
|
"acc_norm_stderr": 0.037184890068181146
|
||
|
|
},
|
||
|
|
"hendrycksTest-global_facts": {
|
||
|
|
"acc": 0.34,
|
||
|
|
"acc_stderr": 0.047609522856952365,
|
||
|
|
"acc_norm": 0.34,
|
||
|
|
"acc_norm_stderr": 0.047609522856952365
|
||
|
|
},
|
||
|
|
"hendrycksTest-high_school_biology": {
|
||
|
|
"acc": 0.4838709677419355,
|
||
|
|
"acc_stderr": 0.028429203176724555,
|
||
|
|
"acc_norm": 0.4838709677419355,
|
||
|
|
"acc_norm_stderr": 0.028429203176724555
|
||
|
|
},
|
||
|
|
"hendrycksTest-high_school_chemistry": {
|
||
|
|
"acc": 0.28078817733990147,
|
||
|
|
"acc_stderr": 0.03161856335358608,
|
||
|
|
"acc_norm": 0.28078817733990147,
|
||
|
|
"acc_norm_stderr": 0.03161856335358608
|
||
|
|
},
|
||
|
|
"hendrycksTest-high_school_computer_science": {
|
||
|
|
"acc": 0.43,
|
||
|
|
"acc_stderr": 0.049756985195624284,
|
||
|
|
"acc_norm": 0.43,
|
||
|
|
"acc_norm_stderr": 0.049756985195624284
|
||
|
|
},
|
||
|
|
"hendrycksTest-high_school_european_history": {
|
||
|
|
"acc": 0.5636363636363636,
|
||
|
|
"acc_stderr": 0.03872592983524754,
|
||
|
|
"acc_norm": 0.5636363636363636,
|
||
|
|
"acc_norm_stderr": 0.03872592983524754
|
||
|
|
},
|
||
|
|
"hendrycksTest-high_school_geography": {
|
||
|
|
"acc": 0.4898989898989899,
|
||
|
|
"acc_stderr": 0.035616254886737454,
|
||
|
|
"acc_norm": 0.4898989898989899,
|
||
|
|
"acc_norm_stderr": 0.035616254886737454
|
||
|
|
},
|
||
|
|
"hendrycksTest-high_school_government_and_politics": {
|
||
|
|
"acc": 0.6787564766839378,
|
||
|
|
"acc_stderr": 0.033699508685490674,
|
||
|
|
"acc_norm": 0.6787564766839378,
|
||
|
|
"acc_norm_stderr": 0.033699508685490674
|
||
|
|
},
|
||
|
|
"hendrycksTest-high_school_macroeconomics": {
|
||
|
|
"acc": 0.4,
|
||
|
|
"acc_stderr": 0.024838811988033165,
|
||
|
|
"acc_norm": 0.4,
|
||
|
|
"acc_norm_stderr": 0.024838811988033165
|
||
|
|
},
|
||
|
|
"hendrycksTest-high_school_mathematics": {
|
||
|
|
"acc": 0.2740740740740741,
|
||
|
|
"acc_stderr": 0.027195934804085622,
|
||
|
|
"acc_norm": 0.2740740740740741,
|
||
|
|
"acc_norm_stderr": 0.027195934804085622
|
||
|
|
},
|
||
|
|
"hendrycksTest-high_school_microeconomics": {
|
||
|
|
"acc": 0.3865546218487395,
|
||
|
|
"acc_stderr": 0.0316314580755238,
|
||
|
|
"acc_norm": 0.3865546218487395,
|
||
|
|
"acc_norm_stderr": 0.0316314580755238
|
||
|
|
},
|
||
|
|
"hendrycksTest-high_school_physics": {
|
||
|
|
"acc": 0.2847682119205298,
|
||
|
|
"acc_stderr": 0.03684881521389023,
|
||
|
|
"acc_norm": 0.2847682119205298,
|
||
|
|
"acc_norm_stderr": 0.03684881521389023
|
||
|
|
},
|
||
|
|
"hendrycksTest-high_school_psychology": {
|
||
|
|
"acc": 0.6073394495412844,
|
||
|
|
"acc_stderr": 0.020937505161201096,
|
||
|
|
"acc_norm": 0.6073394495412844,
|
||
|
|
"acc_norm_stderr": 0.020937505161201096
|
||
|
|
},
|
||
|
|
"hendrycksTest-high_school_statistics": {
|
||
|
|
"acc": 0.4027777777777778,
|
||
|
|
"acc_stderr": 0.03344887382997866,
|
||
|
|
"acc_norm": 0.4027777777777778,
|
||
|
|
"acc_norm_stderr": 0.03344887382997866
|
||
|
|
},
|
||
|
|
"hendrycksTest-high_school_us_history": {
|
||
|
|
"acc": 0.5490196078431373,
|
||
|
|
"acc_stderr": 0.03492406104163613,
|
||
|
|
"acc_norm": 0.5490196078431373,
|
||
|
|
"acc_norm_stderr": 0.03492406104163613
|
||
|
|
},
|
||
|
|
"hendrycksTest-high_school_world_history": {
|
||
|
|
"acc": 0.6244725738396625,
|
||
|
|
"acc_stderr": 0.03152256243091156,
|
||
|
|
"acc_norm": 0.6244725738396625,
|
||
|
|
"acc_norm_stderr": 0.03152256243091156
|
||
|
|
},
|
||
|
|
"hendrycksTest-human_aging": {
|
||
|
|
"acc": 0.48878923766816146,
|
||
|
|
"acc_stderr": 0.033549366530984746,
|
||
|
|
"acc_norm": 0.48878923766816146,
|
||
|
|
"acc_norm_stderr": 0.033549366530984746
|
||
|
|
},
|
||
|
|
"hendrycksTest-human_sexuality": {
|
||
|
|
"acc": 0.4961832061068702,
|
||
|
|
"acc_stderr": 0.04385162325601553,
|
||
|
|
"acc_norm": 0.4961832061068702,
|
||
|
|
"acc_norm_stderr": 0.04385162325601553
|
||
|
|
},
|
||
|
|
"hendrycksTest-international_law": {
|
||
|
|
"acc": 0.6446280991735537,
|
||
|
|
"acc_stderr": 0.04369236326573981,
|
||
|
|
"acc_norm": 0.6446280991735537,
|
||
|
|
"acc_norm_stderr": 0.04369236326573981
|
||
|
|
},
|
||
|
|
"hendrycksTest-jurisprudence": {
|
||
|
|
"acc": 0.48148148148148145,
|
||
|
|
"acc_stderr": 0.04830366024635331,
|
||
|
|
"acc_norm": 0.48148148148148145,
|
||
|
|
"acc_norm_stderr": 0.04830366024635331
|
||
|
|
},
|
||
|
|
"hendrycksTest-logical_fallacies": {
|
||
|
|
"acc": 0.4171779141104294,
|
||
|
|
"acc_stderr": 0.038741028598180814,
|
||
|
|
"acc_norm": 0.4171779141104294,
|
||
|
|
"acc_norm_stderr": 0.038741028598180814
|
||
|
|
},
|
||
|
|
"hendrycksTest-machine_learning": {
|
||
|
|
"acc": 0.3125,
|
||
|
|
"acc_stderr": 0.043994650575715215,
|
||
|
|
"acc_norm": 0.3125,
|
||
|
|
"acc_norm_stderr": 0.043994650575715215
|
||
|
|
},
|
||
|
|
"hendrycksTest-management": {
|
||
|
|
"acc": 0.5242718446601942,
|
||
|
|
"acc_stderr": 0.049449010929737795,
|
||
|
|
"acc_norm": 0.5242718446601942,
|
||
|
|
"acc_norm_stderr": 0.049449010929737795
|
||
|
|
},
|
||
|
|
"hendrycksTest-marketing": {
|
||
|
|
"acc": 0.6581196581196581,
|
||
|
|
"acc_stderr": 0.03107502852650775,
|
||
|
|
"acc_norm": 0.6581196581196581,
|
||
|
|
"acc_norm_stderr": 0.03107502852650775
|
||
|
|
},
|
||
|
|
"hendrycksTest-medical_genetics": {
|
||
|
|
"acc": 0.47,
|
||
|
|
"acc_stderr": 0.050161355804659205,
|
||
|
|
"acc_norm": 0.47,
|
||
|
|
"acc_norm_stderr": 0.050161355804659205
|
||
|
|
},
|
||
|
|
"hendrycksTest-miscellaneous": {
|
||
|
|
"acc": 0.5938697318007663,
|
||
|
|
"acc_stderr": 0.017562037406478916,
|
||
|
|
"acc_norm": 0.5938697318007663,
|
||
|
|
"acc_norm_stderr": 0.017562037406478916
|
||
|
|
},
|
||
|
|
"hendrycksTest-moral_disputes": {
|
||
|
|
"acc": 0.4884393063583815,
|
||
|
|
"acc_stderr": 0.026911898686377906,
|
||
|
|
"acc_norm": 0.4884393063583815,
|
||
|
|
"acc_norm_stderr": 0.026911898686377906
|
||
|
|
},
|
||
|
|
"hendrycksTest-moral_scenarios": {
|
||
|
|
"acc": 0.23798882681564246,
|
||
|
|
"acc_stderr": 0.014242630070574915,
|
||
|
|
"acc_norm": 0.23798882681564246,
|
||
|
|
"acc_norm_stderr": 0.014242630070574915
|
||
|
|
},
|
||
|
|
"hendrycksTest-nutrition": {
|
||
|
|
"acc": 0.5065359477124183,
|
||
|
|
"acc_stderr": 0.028627470550556047,
|
||
|
|
"acc_norm": 0.5065359477124183,
|
||
|
|
"acc_norm_stderr": 0.028627470550556047
|
||
|
|
},
|
||
|
|
"hendrycksTest-philosophy": {
|
||
|
|
"acc": 0.5562700964630225,
|
||
|
|
"acc_stderr": 0.02821768355665231,
|
||
|
|
"acc_norm": 0.5562700964630225,
|
||
|
|
"acc_norm_stderr": 0.02821768355665231
|
||
|
|
},
|
||
|
|
"hendrycksTest-prehistory": {
|
||
|
|
"acc": 0.49382716049382713,
|
||
|
|
"acc_stderr": 0.027818623962583295,
|
||
|
|
"acc_norm": 0.49382716049382713,
|
||
|
|
"acc_norm_stderr": 0.027818623962583295
|
||
|
|
},
|
||
|
|
"hendrycksTest-professional_accounting": {
|
||
|
|
"acc": 0.3404255319148936,
|
||
|
|
"acc_stderr": 0.028267657482650144,
|
||
|
|
"acc_norm": 0.3404255319148936,
|
||
|
|
"acc_norm_stderr": 0.028267657482650144
|
||
|
|
},
|
||
|
|
"hendrycksTest-professional_law": {
|
||
|
|
"acc": 0.3318122555410691,
|
||
|
|
"acc_stderr": 0.012026088259897632,
|
||
|
|
"acc_norm": 0.3318122555410691,
|
||
|
|
"acc_norm_stderr": 0.012026088259897632
|
||
|
|
},
|
||
|
|
"hendrycksTest-professional_medicine": {
|
||
|
|
"acc": 0.4963235294117647,
|
||
|
|
"acc_stderr": 0.030372015885428195,
|
||
|
|
"acc_norm": 0.4963235294117647,
|
||
|
|
"acc_norm_stderr": 0.030372015885428195
|
||
|
|
},
|
||
|
|
"hendrycksTest-professional_psychology": {
|
||
|
|
"acc": 0.39215686274509803,
|
||
|
|
"acc_stderr": 0.01975172650876263,
|
||
|
|
"acc_norm": 0.39215686274509803,
|
||
|
|
"acc_norm_stderr": 0.01975172650876263
|
||
|
|
},
|
||
|
|
"hendrycksTest-public_relations": {
|
||
|
|
"acc": 0.5454545454545454,
|
||
|
|
"acc_stderr": 0.04769300568972743,
|
||
|
|
"acc_norm": 0.5454545454545454,
|
||
|
|
"acc_norm_stderr": 0.04769300568972743
|
||
|
|
},
|
||
|
|
"hendrycksTest-security_studies": {
|
||
|
|
"acc": 0.46530612244897956,
|
||
|
|
"acc_stderr": 0.03193207024425314,
|
||
|
|
"acc_norm": 0.46530612244897956,
|
||
|
|
"acc_norm_stderr": 0.03193207024425314
|
||
|
|
},
|
||
|
|
"hendrycksTest-sociology": {
|
||
|
|
"acc": 0.5671641791044776,
|
||
|
|
"acc_stderr": 0.03503490923673281,
|
||
|
|
"acc_norm": 0.5671641791044776,
|
||
|
|
"acc_norm_stderr": 0.03503490923673281
|
||
|
|
},
|
||
|
|
"hendrycksTest-us_foreign_policy": {
|
||
|
|
"acc": 0.68,
|
||
|
|
"acc_stderr": 0.04688261722621504,
|
||
|
|
"acc_norm": 0.68,
|
||
|
|
"acc_norm_stderr": 0.04688261722621504
|
||
|
|
},
|
||
|
|
"hendrycksTest-virology": {
|
||
|
|
"acc": 0.39156626506024095,
|
||
|
|
"acc_stderr": 0.037998574544796375,
|
||
|
|
"acc_norm": 0.39156626506024095,
|
||
|
|
"acc_norm_stderr": 0.037998574544796375
|
||
|
|
},
|
||
|
|
"hendrycksTest-world_religions": {
|
||
|
|
"acc": 0.6549707602339181,
|
||
|
|
"acc_stderr": 0.03645981377388807,
|
||
|
|
"acc_norm": 0.6549707602339181,
|
||
|
|
"acc_norm_stderr": 0.03645981377388807
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"versions": {
|
||
|
|
"hendrycksTest-abstract_algebra": 1,
|
||
|
|
"hendrycksTest-anatomy": 1,
|
||
|
|
"hendrycksTest-astronomy": 1,
|
||
|
|
"hendrycksTest-business_ethics": 1,
|
||
|
|
"hendrycksTest-clinical_knowledge": 1,
|
||
|
|
"hendrycksTest-college_biology": 1,
|
||
|
|
"hendrycksTest-college_chemistry": 1,
|
||
|
|
"hendrycksTest-college_computer_science": 1,
|
||
|
|
"hendrycksTest-college_mathematics": 1,
|
||
|
|
"hendrycksTest-college_medicine": 1,
|
||
|
|
"hendrycksTest-college_physics": 1,
|
||
|
|
"hendrycksTest-computer_security": 1,
|
||
|
|
"hendrycksTest-conceptual_physics": 1,
|
||
|
|
"hendrycksTest-econometrics": 1,
|
||
|
|
"hendrycksTest-electrical_engineering": 1,
|
||
|
|
"hendrycksTest-elementary_mathematics": 1,
|
||
|
|
"hendrycksTest-formal_logic": 1,
|
||
|
|
"hendrycksTest-global_facts": 1,
|
||
|
|
"hendrycksTest-high_school_biology": 1,
|
||
|
|
"hendrycksTest-high_school_chemistry": 1,
|
||
|
|
"hendrycksTest-high_school_computer_science": 1,
|
||
|
|
"hendrycksTest-high_school_european_history": 1,
|
||
|
|
"hendrycksTest-high_school_geography": 1,
|
||
|
|
"hendrycksTest-high_school_government_and_politics": 1,
|
||
|
|
"hendrycksTest-high_school_macroeconomics": 1,
|
||
|
|
"hendrycksTest-high_school_mathematics": 1,
|
||
|
|
"hendrycksTest-high_school_microeconomics": 1,
|
||
|
|
"hendrycksTest-high_school_physics": 1,
|
||
|
|
"hendrycksTest-high_school_psychology": 1,
|
||
|
|
"hendrycksTest-high_school_statistics": 1,
|
||
|
|
"hendrycksTest-high_school_us_history": 1,
|
||
|
|
"hendrycksTest-high_school_world_history": 1,
|
||
|
|
"hendrycksTest-human_aging": 1,
|
||
|
|
"hendrycksTest-human_sexuality": 1,
|
||
|
|
"hendrycksTest-international_law": 1,
|
||
|
|
"hendrycksTest-jurisprudence": 1,
|
||
|
|
"hendrycksTest-logical_fallacies": 1,
|
||
|
|
"hendrycksTest-machine_learning": 1,
|
||
|
|
"hendrycksTest-management": 1,
|
||
|
|
"hendrycksTest-marketing": 1,
|
||
|
|
"hendrycksTest-medical_genetics": 1,
|
||
|
|
"hendrycksTest-miscellaneous": 1,
|
||
|
|
"hendrycksTest-moral_disputes": 1,
|
||
|
|
"hendrycksTest-moral_scenarios": 1,
|
||
|
|
"hendrycksTest-nutrition": 1,
|
||
|
|
"hendrycksTest-philosophy": 1,
|
||
|
|
"hendrycksTest-prehistory": 1,
|
||
|
|
"hendrycksTest-professional_accounting": 1,
|
||
|
|
"hendrycksTest-professional_law": 1,
|
||
|
|
"hendrycksTest-professional_medicine": 1,
|
||
|
|
"hendrycksTest-professional_psychology": 1,
|
||
|
|
"hendrycksTest-public_relations": 1,
|
||
|
|
"hendrycksTest-security_studies": 1,
|
||
|
|
"hendrycksTest-sociology": 1,
|
||
|
|
"hendrycksTest-us_foreign_policy": 1,
|
||
|
|
"hendrycksTest-virology": 1,
|
||
|
|
"hendrycksTest-world_religions": 1
|
||
|
|
},
|
||
|
|
"config": {
|
||
|
|
"model": "hf-causal-experimental",
|
||
|
|
"model_args": {
|
||
|
|
"pretrained": "/network/alexandre/research/cerebras/retrained_s50",
|
||
|
|
"trust_remote_code": true,
|
||
|
|
"dtype": "bfloat16"
|
||
|
|
},
|
||
|
|
"num_fewshot": 5,
|
||
|
|
"batch_size": "4",
|
||
|
|
"batch_sizes": [],
|
||
|
|
"device": "cuda:0",
|
||
|
|
"no_cache": true,
|
||
|
|
"limit": null,
|
||
|
|
"bootstrap_iters": 100000,
|
||
|
|
"description_dict": {}
|
||
|
|
}
|
||
|
|
}
|