101 lines
2.3 KiB
JSON
101 lines
2.3 KiB
JSON
{
|
|
"benchmark": "MMLU finance and business subset",
|
|
"confidence_interval_95": [
|
|
0.71513558,
|
|
0.77929395
|
|
],
|
|
"dataset": "cais/mmlu",
|
|
"dataset_revision": "c30699e8356da336a370243923dbaf21066bb9fe",
|
|
"device": "mps",
|
|
"dtype": "bfloat16",
|
|
"evaluated_at": "2026-07-29T10:03:49.030340+00:00",
|
|
"few_shot": 5,
|
|
"macro_accuracy": 0.74857143,
|
|
"micro_accuracy": 0.74857143,
|
|
"model": ".artifacts/blum-transformers-v1/portable",
|
|
"model_revision": "ad6f5cec7f729370d2976d8c78983521cb37ca83",
|
|
"official_leaderboard_result": false,
|
|
"sample_limit_per_subject": 100,
|
|
"sample_size": 700,
|
|
"status": "community_evaluation",
|
|
"subjects": [
|
|
{
|
|
"accuracy": 0.77,
|
|
"confidence_interval_95": [
|
|
0.67845617,
|
|
0.84156734
|
|
],
|
|
"correct": 77,
|
|
"duration_seconds": 285.586,
|
|
"sample_size": 100,
|
|
"subject": "business_ethics"
|
|
},
|
|
{
|
|
"accuracy": 0.65,
|
|
"confidence_interval_95": [
|
|
0.55254443,
|
|
0.73635752
|
|
],
|
|
"correct": 65,
|
|
"duration_seconds": 222.028,
|
|
"sample_size": 100,
|
|
"subject": "econometrics"
|
|
},
|
|
{
|
|
"accuracy": 0.7,
|
|
"confidence_interval_95": [
|
|
0.60415145,
|
|
0.78105115
|
|
],
|
|
"correct": 70,
|
|
"duration_seconds": 187.56,
|
|
"sample_size": 100,
|
|
"subject": "high_school_macroeconomics"
|
|
},
|
|
{
|
|
"accuracy": 0.87,
|
|
"confidence_interval_95": [
|
|
0.79019649,
|
|
0.92242833
|
|
],
|
|
"correct": 87,
|
|
"duration_seconds": 171.766,
|
|
"sample_size": 100,
|
|
"subject": "high_school_microeconomics"
|
|
},
|
|
{
|
|
"accuracy": 0.87,
|
|
"confidence_interval_95": [
|
|
0.79019649,
|
|
0.92242833
|
|
],
|
|
"correct": 87,
|
|
"duration_seconds": 101.632,
|
|
"sample_size": 100,
|
|
"subject": "management"
|
|
},
|
|
{
|
|
"accuracy": 0.92,
|
|
"confidence_interval_95": [
|
|
0.85001892,
|
|
0.95890654
|
|
],
|
|
"correct": 92,
|
|
"duration_seconds": 194.458,
|
|
"sample_size": 100,
|
|
"subject": "marketing"
|
|
},
|
|
{
|
|
"accuracy": 0.46,
|
|
"confidence_interval_95": [
|
|
0.36560808,
|
|
0.5573514
|
|
],
|
|
"correct": 46,
|
|
"duration_seconds": 543.284,
|
|
"sample_size": 100,
|
|
"subject": "professional_accounting"
|
|
}
|
|
]
|
|
}
|