130 lines
2.9 KiB
JSON
130 lines
2.9 KiB
JSON
|
{
|
||
|
|
"criterion": "0.5 * language_accuracy + 0.5 * culture_accuracy",
|
||
|
|
"tie_breakers": [
|
||
|
|
"fewer unparsed answers",
|
||
|
|
"earlier checkpoint"
|
||
|
|
],
|
||
|
|
"training_pool": "data/prompts_200k_clean.jsonl",
|
||
|
|
"training_pool_sha256": "d679f3e79221aea95c64ef0be177530b02dce25d4745b0247cbae7b677f1ceab",
|
||
|
|
"validation_pool": "data/prompts_holdout_clean.jsonl",
|
||
|
|
"validation_pool_sha256": "749dbcdea8b244c3609acecddf618d9780559f501da18b09f650dfe10f2fb881",
|
||
|
|
"evaluation_steps": [
|
||
|
|
1692,
|
||
|
|
3384,
|
||
|
|
5076,
|
||
|
|
6768
|
||
|
|
],
|
||
|
|
"evaluations": [
|
||
|
|
{
|
||
|
|
"step": 1692,
|
||
|
|
"rows": 6745,
|
||
|
|
"correct": 3224,
|
||
|
|
"accuracy": 0.47798369162342474,
|
||
|
|
"balanced_accuracy": 0.5274351150178963,
|
||
|
|
"macro": {
|
||
|
|
"lang": {
|
||
|
|
"correct": 716,
|
||
|
|
"total": 1186,
|
||
|
|
"accuracy": 0.6037099494097807
|
||
|
|
},
|
||
|
|
"culture": {
|
||
|
|
"correct": 2508,
|
||
|
|
"total": 5559,
|
||
|
|
"accuracy": 0.4511602806260119
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"unparsed": 0,
|
||
|
|
"prediction_distribution": {
|
||
|
|
"A": 2534,
|
||
|
|
"B": 1649,
|
||
|
|
"C": 1840,
|
||
|
|
"D": 630,
|
||
|
|
"E": 92
|
||
|
|
}
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"step": 3384,
|
||
|
|
"rows": 6745,
|
||
|
|
"correct": 3510,
|
||
|
|
"accuracy": 0.5203854707190512,
|
||
|
|
"balanced_accuracy": 0.5733892625695172,
|
||
|
|
"macro": {
|
||
|
|
"lang": {
|
||
|
|
"correct": 777,
|
||
|
|
"total": 1186,
|
||
|
|
"accuracy": 0.6551433389544689
|
||
|
|
},
|
||
|
|
"culture": {
|
||
|
|
"correct": 2733,
|
||
|
|
"total": 5559,
|
||
|
|
"accuracy": 0.4916351861845656
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"unparsed": 0,
|
||
|
|
"prediction_distribution": {
|
||
|
|
"A": 809,
|
||
|
|
"B": 2210,
|
||
|
|
"C": 2388,
|
||
|
|
"D": 1310,
|
||
|
|
"E": 28
|
||
|
|
}
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"step": 5076,
|
||
|
|
"rows": 6745,
|
||
|
|
"correct": 3727,
|
||
|
|
"accuracy": 0.5525574499629355,
|
||
|
|
"balanced_accuracy": 0.6101524896048429,
|
||
|
|
"macro": {
|
||
|
|
"lang": {
|
||
|
|
"correct": 829,
|
||
|
|
"total": 1186,
|
||
|
|
"accuracy": 0.6989881956155143
|
||
|
|
},
|
||
|
|
"culture": {
|
||
|
|
"correct": 2898,
|
||
|
|
"total": 5559,
|
||
|
|
"accuracy": 0.5213167835941717
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"unparsed": 0,
|
||
|
|
"prediction_distribution": {
|
||
|
|
"A": 1529,
|
||
|
|
"B": 1649,
|
||
|
|
"C": 2189,
|
||
|
|
"D": 1270,
|
||
|
|
"E": 108
|
||
|
|
}
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"step": 6768,
|
||
|
|
"rows": 6745,
|
||
|
|
"correct": 3775,
|
||
|
|
"accuracy": 0.5596738324684952,
|
||
|
|
"balanced_accuracy": 0.6138065310131664,
|
||
|
|
"macro": {
|
||
|
|
"lang": {
|
||
|
|
"correct": 827,
|
||
|
|
"total": 1186,
|
||
|
|
"accuracy": 0.6973018549747049
|
||
|
|
},
|
||
|
|
"culture": {
|
||
|
|
"correct": 2948,
|
||
|
|
"total": 5559,
|
||
|
|
"accuracy": 0.530311207051628
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"unparsed": 0,
|
||
|
|
"prediction_distribution": {
|
||
|
|
"A": 1625,
|
||
|
|
"B": 1856,
|
||
|
|
"C": 1861,
|
||
|
|
"D": 1332,
|
||
|
|
"E": 71
|
||
|
|
}
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"selected_step": 6768,
|
||
|
|
"selected_balanced_accuracy": 0.6138065310131664
|
||
|
|
}
|