{
  "sphragis_version": "0.1.0",
  "generated_at": "2026-07-27T19:49:25.414448+00:00",
  "reference": {
    "endpoint": "http://127.0.0.1:8080/v1",
    "model": "ref"
  },
  "candidate": {
    "endpoint": "http://127.0.0.1:8081/v1",
    "model": "cand"
  },
  "pack": {
    "name": "/root/pack_v2.jsonl",
    "n_tasks": 2800,
    "sha256": "2de27099bbb15bab4f7b35599b215f038fdb68b3f5f4e1cda1a464f9dd18e14e"
  },
  "overall": "FAIL",
  "claim": "At least one axis shows a statistically significant accuracy regression (exact McNemar, Holm-corrected, alpha=0.05).",
  "axes": [
    {
      "axis": "arithmetic",
      "n": 1400,
      "counts": {
        "both_correct": 970,
        "ref_only": 117,
        "cand_only": 74,
        "both_wrong": 239
      },
      "acc_reference": 0.776429,
      "acc_candidate": 0.745714,
      "regression_d": 0.030714,
      "d_ci": [
        0.011429,
        0.05
      ],
      "d_upper_bound": 0.047143,
      "p_regression": 0.00114373,
      "p_regression_holm": 0.00457492,
      "p_improvement": 0.99930487,
      "improved": false,
      "mde_at_power": 0.024527,
      "n_needed_for_margin": 936,
      "verdict": "FAIL",
      "reason": "significant_regression"
    },
    {
      "axis": "instruction",
      "n": 600,
      "counts": {
        "both_correct": 274,
        "ref_only": 30,
        "cand_only": 59,
        "both_wrong": 237
      },
      "acc_reference": 0.506667,
      "acc_candidate": 0.555,
      "regression_d": -0.048333,
      "d_ci": [
        -0.078333,
        -0.018333
      ],
      "d_upper_bound": -0.023333,
      "p_regression": 0.99933237,
      "p_regression_holm": 1.0,
      "p_improvement": 0.00139263,
      "improved": true,
      "mde_at_power": 0.039027,
      "n_needed_for_margin": 1017,
      "verdict": "PASS",
      "reason": "non_inferior_within_margin"
    },
    {
      "axis": "knowledge",
      "n": 400,
      "counts": {
        "both_correct": 326,
        "ref_only": 25,
        "cand_only": 20,
        "both_wrong": 29
      },
      "acc_reference": 0.8775,
      "acc_candidate": 0.865,
      "regression_d": 0.0125,
      "d_ci": [
        -0.02,
        0.045
      ],
      "d_upper_bound": 0.04,
      "p_regression": 0.27574216,
      "p_regression_holm": 0.82722649,
      "p_improvement": 0.81435098,
      "improved": false,
      "mde_at_power": 0.041591,
      "n_needed_for_margin": 771,
      "verdict": "INCONCLUSIVE",
      "reason": "ci_too_wide"
    },
    {
      "axis": "reasoning",
      "n": 400,
      "counts": {
        "both_correct": 256,
        "ref_only": 7,
        "cand_only": 79,
        "both_wrong": 58
      },
      "acc_reference": 0.6575,
      "acc_candidate": 0.8375,
      "regression_d": -0.18,
      "d_ci": [
        -0.2225,
        -0.1375
      ],
      "d_upper_bound": -0.145,
      "p_regression": 1.0,
      "p_regression_holm": 1.0,
      "p_improvement": 0.0,
      "improved": true,
      "mde_at_power": 0.057496,
      "n_needed_for_margin": 1475,
      "verdict": "PASS",
      "reason": "non_inferior_within_margin"
    }
  ],
  "params": {
    "margin": 0.03,
    "alpha": 0.05,
    "n_floor": 30,
    "power": 0.8,
    "n_boot": 4000,
    "seed": 0
  }
}