{
  "benchmark_date": "2026-06-09",
  "summary": "Local train-only Korean semantic sentinel diagnostics. These are not official leaderboard scores.",
  "jumini_internal_scores_file": "jumini_internal_scores_20260609.json",
  "main_public_comparison": {
    "metric": "Korean semantic pass@33",
    "prompt_set": "/tmp/chojm_hw2_runs/data/clean7020_passk32_accepted_imr_20260609/sentinel_imr_30x3_90.jsonl",
    "decoding": {
      "include_greedy_rollout": true,
      "samples_per_prompt": 32,
      "temperature": 0.7,
      "top_p": 0.95,
      "top_k": 50,
      "max_new_tokens": 64
    },
    "models": {
      "Jumini-Ko-1.2B-clean7020": {
        "params": 1260505088,
        "language_focus": "Korean from-scratch",
        "summary_json": "/tmp/chojm_hw2_runs/passk_basin_inventory_20260609/clean7020_imr_k32_gpu2/passk_summary.json",
        "pass_any": "38/90",
        "pass_any_score": 42.22222222222222,
        "pass_rollouts": "131/2884",
        "pass_rollout_score": 4.542302357836339,
        "greedy_hit": "0/90",
        "categories": {
          "instruction": "26/30",
          "math": "8/30",
          "reasoning": "4/30"
        }
      },
      "TinyLlama-1.1B-intermediate-step-1431k-3T": {
        "repo_id": "TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T",
        "params": 1100000000,
        "language_focus": "English general base",
        "summary_json": "/tmp/chojm_hw2_runs/hf_baseline_compare_20260609/tinyllama_1p1b_base_k32/summary.json",
        "pass_any": "31/90",
        "pass_any_score": 34.44444444444444,
        "pass_rollouts": "56/2966",
        "pass_rollout_score": 1.8880647336480108,
        "greedy_hit": "0/90",
        "categories": {
          "instruction": "27/30",
          "math": "4/30",
          "reasoning": "0/30"
        }
      }
    }
  },
  "public_refresh_evidence": {
    "metric": "Stratified Korean semantic pass@k",
    "previous_hf_v3a6200": {
      "summary_json": "/tmp/chojm_hw2_runs/passk_basin_inventory_20260609/v3a6200_strat240/passk_summary.json",
      "pass_any": "0/240",
      "pass_any_score": 0.0,
      "pass_rollouts": "0/2094",
      "pass_rollout_score": 0.0
    },
    "current_clean7020": {
      "summary_json": "/tmp/chojm_hw2_runs/passk_basin_inventory_20260609/clean7020_strat240_gpu2/passk_summary.json",
      "pass_any": "21/240",
      "pass_any_score": 8.75,
      "pass_rollouts": "29/2105",
      "pass_rollout_score": 1.3776722090261282
    }
  }
}