133 lines
3.2 KiB
JSON
133 lines
3.2 KiB
JSON
{
|
|
"sphragis_version": "0.1.0",
|
|
"generated_at": "2026-07-27T11:21:13.795774+00:00",
|
|
"reference": {
|
|
"endpoint": "http://127.0.0.1:8080/v1",
|
|
"model": "ref"
|
|
},
|
|
"candidate": {
|
|
"endpoint": "http://127.0.0.1:8081/v1",
|
|
"model": "cand"
|
|
},
|
|
"pack": {
|
|
"name": "/root/pack_v2.jsonl",
|
|
"n_tasks": 2800,
|
|
"sha256": "2de27099bbb15bab4f7b35599b215f038fdb68b3f5f4e1cda1a464f9dd18e14e"
|
|
},
|
|
"overall": "FAIL",
|
|
"claim": "At least one axis shows a statistically significant accuracy regression (exact McNemar, Holm-corrected, alpha=0.05).",
|
|
"axes": [
|
|
{
|
|
"axis": "arithmetic",
|
|
"n": 1400,
|
|
"counts": {
|
|
"both_correct": 1208,
|
|
"ref_only": 22,
|
|
"cand_only": 9,
|
|
"both_wrong": 161
|
|
},
|
|
"acc_reference": 0.878571,
|
|
"acc_candidate": 0.869286,
|
|
"regression_d": 0.009286,
|
|
"d_ci": [
|
|
0.001429,
|
|
0.017143
|
|
],
|
|
"d_upper_bound": 0.015714,
|
|
"p_regression": 0.01472469,
|
|
"p_regression_holm": 0.04417406,
|
|
"p_improvement": 0.99466308,
|
|
"improved": false,
|
|
"mde_at_power": 0.009881,
|
|
"n_needed_for_margin": 204,
|
|
"verdict": "FAIL",
|
|
"reason": "significant_regression"
|
|
},
|
|
{
|
|
"axis": "instruction",
|
|
"n": 600,
|
|
"counts": {
|
|
"both_correct": 508,
|
|
"ref_only": 8,
|
|
"cand_only": 0,
|
|
"both_wrong": 84
|
|
},
|
|
"acc_reference": 0.86,
|
|
"acc_candidate": 0.846667,
|
|
"regression_d": 0.013333,
|
|
"d_ci": [
|
|
0.005,
|
|
0.023333
|
|
],
|
|
"d_upper_bound": 0.021667,
|
|
"p_regression": 0.00390625,
|
|
"p_regression_holm": 0.015625,
|
|
"p_improvement": 1.0,
|
|
"improved": false,
|
|
"mde_at_power": 0.011701,
|
|
"n_needed_for_margin": 204,
|
|
"verdict": "FAIL",
|
|
"reason": "significant_regression"
|
|
},
|
|
{
|
|
"axis": "knowledge",
|
|
"n": 400,
|
|
"counts": {
|
|
"both_correct": 367,
|
|
"ref_only": 7,
|
|
"cand_only": 2,
|
|
"both_wrong": 24
|
|
},
|
|
"acc_reference": 0.935,
|
|
"acc_candidate": 0.9225,
|
|
"regression_d": 0.0125,
|
|
"d_ci": [
|
|
-0.0025,
|
|
0.0275
|
|
],
|
|
"d_upper_bound": 0.025,
|
|
"p_regression": 0.08984375,
|
|
"p_regression_holm": 0.1796875,
|
|
"p_improvement": 0.98046875,
|
|
"improved": false,
|
|
"mde_at_power": 0.0186,
|
|
"n_needed_for_margin": 204,
|
|
"verdict": "PASS",
|
|
"reason": "non_inferior_within_margin"
|
|
},
|
|
{
|
|
"axis": "reasoning",
|
|
"n": 400,
|
|
"counts": {
|
|
"both_correct": 328,
|
|
"ref_only": 2,
|
|
"cand_only": 5,
|
|
"both_wrong": 65
|
|
},
|
|
"acc_reference": 0.825,
|
|
"acc_candidate": 0.8325,
|
|
"regression_d": -0.0075,
|
|
"d_ci": [
|
|
-0.02,
|
|
0.005
|
|
],
|
|
"d_upper_bound": 0.0025,
|
|
"p_regression": 0.9375,
|
|
"p_regression_holm": 0.9375,
|
|
"p_improvement": 0.2265625,
|
|
"improved": false,
|
|
"mde_at_power": 0.016404,
|
|
"n_needed_for_margin": 204,
|
|
"verdict": "PASS",
|
|
"reason": "non_inferior_within_margin"
|
|
}
|
|
],
|
|
"params": {
|
|
"margin": 0.03,
|
|
"alpha": 0.05,
|
|
"n_floor": 30,
|
|
"power": 0.8,
|
|
"n_boot": 4000,
|
|
"seed": 0
|
|
}
|
|
} |