{
  "criterion": "0.5 * language_accuracy + 0.5 * culture_accuracy",
  "tie_breakers": [
    "fewer unparsed answers",
    "earlier checkpoint"
  ],
  "training_pool": "data/prompts_200k_clean.jsonl",
  "training_pool_sha256": "d679f3e79221aea95c64ef0be177530b02dce25d4745b0247cbae7b677f1ceab",
  "validation_pool": "data/prompts_holdout_clean.jsonl",
  "validation_pool_sha256": "749dbcdea8b244c3609acecddf618d9780559f501da18b09f650dfe10f2fb881",
  "evaluation_steps": [
    1692,
    3384,
    5076,
    6768
  ],
  "evaluations": [
    {
      "step": 1692,
      "rows": 6745,
      "correct": 3224,
      "accuracy": 0.47798369162342474,
      "balanced_accuracy": 0.5274351150178963,
      "macro": {
        "lang": {
          "correct": 716,
          "total": 1186,
          "accuracy": 0.6037099494097807
        },
        "culture": {
          "correct": 2508,
          "total": 5559,
          "accuracy": 0.4511602806260119
        }
      },
      "unparsed": 0,
      "prediction_distribution": {
        "A": 2534,
        "B": 1649,
        "C": 1840,
        "D": 630,
        "E": 92
      }
    },
    {
      "step": 3384,
      "rows": 6745,
      "correct": 3510,
      "accuracy": 0.5203854707190512,
      "balanced_accuracy": 0.5733892625695172,
      "macro": {
        "lang": {
          "correct": 777,
          "total": 1186,
          "accuracy": 0.6551433389544689
        },
        "culture": {
          "correct": 2733,
          "total": 5559,
          "accuracy": 0.4916351861845656
        }
      },
      "unparsed": 0,
      "prediction_distribution": {
        "A": 809,
        "B": 2210,
        "C": 2388,
        "D": 1310,
        "E": 28
      }
    },
    {
      "step": 5076,
      "rows": 6745,
      "correct": 3727,
      "accuracy": 0.5525574499629355,
      "balanced_accuracy": 0.6101524896048429,
      "macro": {
        "lang": {
          "correct": 829,
          "total": 1186,
          "accuracy": 0.6989881956155143
        },
        "culture": {
          "correct": 2898,
          "total": 5559,
          "accuracy": 0.5213167835941717
        }
      },
      "unparsed": 0,
      "prediction_distribution": {
        "A": 1529,
        "B": 1649,
        "C": 2189,
        "D": 1270,
        "E": 108
      }
    },
    {
      "step": 6768,
      "rows": 6745,
      "correct": 3775,
      "accuracy": 0.5596738324684952,
      "balanced_accuracy": 0.6138065310131664,
      "macro": {
        "lang": {
          "correct": 827,
          "total": 1186,
          "accuracy": 0.6973018549747049
        },
        "culture": {
          "correct": 2948,
          "total": 5559,
          "accuracy": 0.530311207051628
        }
      },
      "unparsed": 0,
      "prediction_distribution": {
        "A": 1625,
        "B": 1856,
        "C": 1861,
        "D": 1332,
        "E": 71
      }
    }
  ],
  "selected_step": 6768,
  "selected_balanced_accuracy": 0.6138065310131664
}