133 lines
3.1 KiB
JSON
133 lines
3.1 KiB
JSON
|
{
|
||
|
|
"sphragis_version": "0.1.0",
|
||
|
|
"generated_at": "2026-07-27T18:34:00.914337+00:00",
|
||
|
|
"reference": {
|
||
|
|
"endpoint": "http://127.0.0.1:8080/v1",
|
||
|
|
"model": "ref"
|
||
|
|
},
|
||
|
|
"candidate": {
|
||
|
|
"endpoint": "http://127.0.0.1:8081/v1",
|
||
|
|
"model": "cand"
|
||
|
|
},
|
||
|
|
"pack": {
|
||
|
|
"name": "/root/pack_v2.jsonl",
|
||
|
|
"n_tasks": 2800,
|
||
|
|
"sha256": "2de27099bbb15bab4f7b35599b215f038fdb68b3f5f4e1cda1a464f9dd18e14e"
|
||
|
|
},
|
||
|
|
"overall": "FAIL",
|
||
|
|
"claim": "At least one axis shows a statistically significant accuracy regression (exact McNemar, Holm-corrected, alpha=0.05).",
|
||
|
|
"axes": [
|
||
|
|
{
|
||
|
|
"axis": "arithmetic",
|
||
|
|
"n": 1400,
|
||
|
|
"counts": {
|
||
|
|
"both_correct": 393,
|
||
|
|
"ref_only": 53,
|
||
|
|
"cand_only": 28,
|
||
|
|
"both_wrong": 926
|
||
|
|
},
|
||
|
|
"acc_reference": 0.318571,
|
||
|
|
"acc_candidate": 0.300714,
|
||
|
|
"regression_d": 0.017857,
|
||
|
|
"d_ci": [
|
||
|
|
0.005,
|
||
|
|
0.030714
|
||
|
|
],
|
||
|
|
"d_upper_bound": 0.028571,
|
||
|
|
"p_regression": 0.00363775,
|
||
|
|
"p_regression_holm": 0.01091325,
|
||
|
|
"p_improvement": 0.99820183,
|
||
|
|
"improved": false,
|
||
|
|
"mde_at_power": 0.015973,
|
||
|
|
"n_needed_for_margin": 396,
|
||
|
|
"verdict": "FAIL",
|
||
|
|
"reason": "significant_regression"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"axis": "instruction",
|
||
|
|
"n": 600,
|
||
|
|
"counts": {
|
||
|
|
"both_correct": 45,
|
||
|
|
"ref_only": 0,
|
||
|
|
"cand_only": 0,
|
||
|
|
"both_wrong": 555
|
||
|
|
},
|
||
|
|
"acc_reference": 0.075,
|
||
|
|
"acc_candidate": 0.075,
|
||
|
|
"regression_d": 0.0,
|
||
|
|
"d_ci": [
|
||
|
|
0.0,
|
||
|
|
0.0
|
||
|
|
],
|
||
|
|
"d_upper_bound": 0.0,
|
||
|
|
"p_regression": 1.0,
|
||
|
|
"p_regression_holm": 1.0,
|
||
|
|
"p_improvement": 1.0,
|
||
|
|
"improved": false,
|
||
|
|
"mde_at_power": null,
|
||
|
|
"n_needed_for_margin": 204,
|
||
|
|
"verdict": "PASS",
|
||
|
|
"reason": "non_inferior_within_margin"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"axis": "knowledge",
|
||
|
|
"n": 400,
|
||
|
|
"counts": {
|
||
|
|
"both_correct": 67,
|
||
|
|
"ref_only": 8,
|
||
|
|
"cand_only": 1,
|
||
|
|
"both_wrong": 324
|
||
|
|
},
|
||
|
|
"acc_reference": 0.1875,
|
||
|
|
"acc_candidate": 0.17,
|
||
|
|
"regression_d": 0.0175,
|
||
|
|
"d_ci": [
|
||
|
|
0.005,
|
||
|
|
0.0325
|
||
|
|
],
|
||
|
|
"d_upper_bound": 0.03,
|
||
|
|
"p_regression": 0.01953125,
|
||
|
|
"p_regression_holm": 0.0390625,
|
||
|
|
"p_improvement": 0.99804688,
|
||
|
|
"improved": false,
|
||
|
|
"mde_at_power": 0.0186,
|
||
|
|
"n_needed_for_margin": 204,
|
||
|
|
"verdict": "FAIL",
|
||
|
|
"reason": "significant_regression"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"axis": "reasoning",
|
||
|
|
"n": 400,
|
||
|
|
"counts": {
|
||
|
|
"both_correct": 128,
|
||
|
|
"ref_only": 35,
|
||
|
|
"cand_only": 8,
|
||
|
|
"both_wrong": 229
|
||
|
|
},
|
||
|
|
"acc_reference": 0.4075,
|
||
|
|
"acc_candidate": 0.34,
|
||
|
|
"regression_d": 0.0675,
|
||
|
|
"d_ci": [
|
||
|
|
0.0375,
|
||
|
|
0.1
|
||
|
|
],
|
||
|
|
"d_upper_bound": 0.095,
|
||
|
|
"p_regression": 2.097e-05,
|
||
|
|
"p_regression_holm": 8.387e-05,
|
||
|
|
"p_improvement": 0.99999552,
|
||
|
|
"improved": false,
|
||
|
|
"mde_at_power": 0.040656,
|
||
|
|
"n_needed_for_margin": 737,
|
||
|
|
"verdict": "FAIL",
|
||
|
|
"reason": "significant_regression"
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"params": {
|
||
|
|
"margin": 0.03,
|
||
|
|
"alpha": 0.05,
|
||
|
|
"n_floor": 30,
|
||
|
|
"power": 0.8,
|
||
|
|
"n_boot": 4000,
|
||
|
|
"seed": 0
|
||
|
|
}
|
||
|
|
}
|