{
  "benchmark": "MMLU finance and business subset",
  "confidence_interval_95": [
    0.71513558,
    0.77929395
  ],
  "dataset": "cais/mmlu",
  "dataset_revision": "c30699e8356da336a370243923dbaf21066bb9fe",
  "device": "mps",
  "dtype": "bfloat16",
  "evaluated_at": "2026-07-29T10:03:49.030340+00:00",
  "few_shot": 5,
  "macro_accuracy": 0.74857143,
  "micro_accuracy": 0.74857143,
  "model": ".artifacts/blum-transformers-v1/portable",
  "model_revision": "ad6f5cec7f729370d2976d8c78983521cb37ca83",
  "official_leaderboard_result": false,
  "sample_limit_per_subject": 100,
  "sample_size": 700,
  "status": "community_evaluation",
  "subjects": [
    {
      "accuracy": 0.77,
      "confidence_interval_95": [
        0.67845617,
        0.84156734
      ],
      "correct": 77,
      "duration_seconds": 285.586,
      "sample_size": 100,
      "subject": "business_ethics"
    },
    {
      "accuracy": 0.65,
      "confidence_interval_95": [
        0.55254443,
        0.73635752
      ],
      "correct": 65,
      "duration_seconds": 222.028,
      "sample_size": 100,
      "subject": "econometrics"
    },
    {
      "accuracy": 0.7,
      "confidence_interval_95": [
        0.60415145,
        0.78105115
      ],
      "correct": 70,
      "duration_seconds": 187.56,
      "sample_size": 100,
      "subject": "high_school_macroeconomics"
    },
    {
      "accuracy": 0.87,
      "confidence_interval_95": [
        0.79019649,
        0.92242833
      ],
      "correct": 87,
      "duration_seconds": 171.766,
      "sample_size": 100,
      "subject": "high_school_microeconomics"
    },
    {
      "accuracy": 0.87,
      "confidence_interval_95": [
        0.79019649,
        0.92242833
      ],
      "correct": 87,
      "duration_seconds": 101.632,
      "sample_size": 100,
      "subject": "management"
    },
    {
      "accuracy": 0.92,
      "confidence_interval_95": [
        0.85001892,
        0.95890654
      ],
      "correct": 92,
      "duration_seconds": 194.458,
      "sample_size": 100,
      "subject": "marketing"
    },
    {
      "accuracy": 0.46,
      "confidence_interval_95": [
        0.36560808,
        0.5573514
      ],
      "correct": 46,
      "duration_seconds": 543.284,
      "sample_size": 100,
      "subject": "professional_accounting"
    }
  ]
}