213 lines
4.9 KiB
JSON
213 lines
4.9 KiB
JSON
{
|
|
"candidate": "l20-edu-135m-deepthin",
|
|
"tasks": {
|
|
"lambada_openai": {
|
|
"metric": "acc,none",
|
|
"value": 0.25402678051620414
|
|
},
|
|
"hellaswag": {
|
|
"metric": "acc_norm,none",
|
|
"value": 0.3272256522605059
|
|
},
|
|
"piqa": {
|
|
"metric": "acc_norm,none",
|
|
"value": 0.6224156692056583
|
|
},
|
|
"arc_easy": {
|
|
"metric": "acc_norm,none",
|
|
"value": 0.5058922558922558
|
|
},
|
|
"arc_challenge": {
|
|
"metric": "acc_norm,none",
|
|
"value": 0.2764505119453925
|
|
},
|
|
"winogrande": {
|
|
"metric": "acc,none",
|
|
"value": 0.5098658247829518
|
|
}
|
|
},
|
|
"baselines": {
|
|
"gpt2-small": {
|
|
"lambada_openai": {
|
|
"metric": "acc,none",
|
|
"value": 0.30758781292451
|
|
},
|
|
"hellaswag": {
|
|
"metric": "acc_norm,none",
|
|
"value": 0.31378211511651066
|
|
},
|
|
"piqa": {
|
|
"metric": "acc_norm,none",
|
|
"value": 0.6207834602829162
|
|
},
|
|
"arc_easy": {
|
|
"metric": "acc_norm,none",
|
|
"value": 0.39730639730639733
|
|
},
|
|
"arc_challenge": {
|
|
"metric": "acc_norm,none",
|
|
"value": 0.22610921501706485
|
|
},
|
|
"winogrande": {
|
|
"metric": "acc,none",
|
|
"value": 0.5067087608524072
|
|
}
|
|
},
|
|
"opt-125m": {
|
|
"lambada_openai": {
|
|
"metric": "acc,none",
|
|
"value": 0.3856006209974772
|
|
},
|
|
"hellaswag": {
|
|
"metric": "acc_norm,none",
|
|
"value": 0.31597291376219877
|
|
},
|
|
"piqa": {
|
|
"metric": "acc_norm,none",
|
|
"value": 0.6202393906420022
|
|
},
|
|
"arc_easy": {
|
|
"metric": "acc_norm,none",
|
|
"value": 0.398989898989899
|
|
},
|
|
"arc_challenge": {
|
|
"metric": "acc_norm,none",
|
|
"value": 0.22098976109215018
|
|
},
|
|
"winogrande": {
|
|
"metric": "acc,none",
|
|
"value": 0.5177584846093133
|
|
}
|
|
},
|
|
"gpt-neo-125m": {
|
|
"lambada_openai": {
|
|
"metric": "acc,none",
|
|
"value": 0.37647972055113527
|
|
},
|
|
"hellaswag": {
|
|
"metric": "acc_norm,none",
|
|
"value": 0.30551682931686913
|
|
},
|
|
"piqa": {
|
|
"metric": "acc_norm,none",
|
|
"value": 0.6213275299238302
|
|
},
|
|
"arc_easy": {
|
|
"metric": "acc_norm,none",
|
|
"value": 0.39646464646464646
|
|
},
|
|
"arc_challenge": {
|
|
"metric": "acc_norm,none",
|
|
"value": 0.23208191126279865
|
|
},
|
|
"winogrande": {
|
|
"metric": "acc,none",
|
|
"value": 0.5098658247829518
|
|
}
|
|
},
|
|
"cerebras-gpt-111m": {
|
|
"lambada_openai": {
|
|
"metric": "acc,none",
|
|
"value": 0.19115078594993207
|
|
},
|
|
"hellaswag": {
|
|
"metric": "acc_norm,none",
|
|
"value": 0.2719577773351922
|
|
},
|
|
"piqa": {
|
|
"metric": "acc_norm,none",
|
|
"value": 0.5810663764961915
|
|
},
|
|
"arc_easy": {
|
|
"metric": "acc_norm,none",
|
|
"value": 0.35058922558922556
|
|
},
|
|
"arc_challenge": {
|
|
"metric": "acc_norm,none",
|
|
"value": 0.2098976109215017
|
|
},
|
|
"winogrande": {
|
|
"metric": "acc,none",
|
|
"value": 0.49013417521704816
|
|
}
|
|
},
|
|
"pythia-160m": {
|
|
"lambada_openai": {
|
|
"metric": "acc,none",
|
|
"value": 0.12245294003493111
|
|
},
|
|
"hellaswag": {
|
|
"metric": "acc_norm,none",
|
|
"value": 0.30302728540131446
|
|
},
|
|
"piqa": {
|
|
"metric": "acc_norm,none",
|
|
"value": 0.5979325353645266
|
|
},
|
|
"arc_easy": {
|
|
"metric": "acc_norm,none",
|
|
"value": 0.3640572390572391
|
|
},
|
|
"arc_challenge": {
|
|
"metric": "acc_norm,none",
|
|
"value": 0.23122866894197952
|
|
},
|
|
"winogrande": {
|
|
"metric": "acc,none",
|
|
"value": 0.5074980268350434
|
|
}
|
|
},
|
|
"smollm-135m": {
|
|
"lambada_openai": {
|
|
"metric": "acc,none",
|
|
"value": 0.3757034737046381
|
|
},
|
|
"hellaswag": {
|
|
"metric": "acc_norm,none",
|
|
"value": 0.42650866361282613
|
|
},
|
|
"piqa": {
|
|
"metric": "acc_norm,none",
|
|
"value": 0.6822633297062024
|
|
},
|
|
"arc_easy": {
|
|
"metric": "acc_norm,none",
|
|
"value": 0.561026936026936
|
|
},
|
|
"arc_challenge": {
|
|
"metric": "acc_norm,none",
|
|
"value": 0.28754266211604096
|
|
},
|
|
"winogrande": {
|
|
"metric": "acc,none",
|
|
"value": 0.5272296764009471
|
|
}
|
|
},
|
|
"smollm2-135m": {
|
|
"lambada_openai": {
|
|
"metric": "acc,none",
|
|
"value": 0.4288763826896953
|
|
},
|
|
"hellaswag": {
|
|
"metric": "acc_norm,none",
|
|
"value": 0.43009360685122483
|
|
},
|
|
"piqa": {
|
|
"metric": "acc_norm,none",
|
|
"value": 0.6838955386289445
|
|
},
|
|
"arc_easy": {
|
|
"metric": "acc_norm,none",
|
|
"value": 0.5854377104377104
|
|
},
|
|
"arc_challenge": {
|
|
"metric": "acc_norm,none",
|
|
"value": 0.29692832764505117
|
|
},
|
|
"winogrande": {
|
|
"metric": "acc,none",
|
|
"value": 0.5248618784530387
|
|
}
|
|
}
|
|
}
|
|
} |