267 lines
5.8 KiB
JSON
267 lines
5.8 KiB
JSON
{
|
|
"by_runner_family": {
|
|
"full_weight_transformers": {
|
|
"json_multistep": {
|
|
"n": 5,
|
|
"pass_rate": 0.8,
|
|
"pass_cols": [
|
|
"schema_ok",
|
|
"checks_consistent_ok",
|
|
"stop_semantics_ok",
|
|
"oracle_equiv_ok"
|
|
],
|
|
"averages": {
|
|
"bucket_score": 2.2
|
|
},
|
|
"top_failures": [
|
|
{
|
|
"reason": "oracle_equiv_ok",
|
|
"count": 1
|
|
}
|
|
],
|
|
"signal_rates": {
|
|
"schema_ok": 1.0,
|
|
"checks_consistent_ok": 1.0,
|
|
"stop_semantics_ok": 1.0,
|
|
"oracle_equiv_ok": 0.8
|
|
},
|
|
"signal_counts": {
|
|
"schema_ok": {
|
|
"present": 5,
|
|
"ok": 5,
|
|
"fail": 0
|
|
},
|
|
"checks_consistent_ok": {
|
|
"present": 5,
|
|
"ok": 5,
|
|
"fail": 0
|
|
},
|
|
"stop_semantics_ok": {
|
|
"present": 5,
|
|
"ok": 5,
|
|
"fail": 0
|
|
},
|
|
"oracle_equiv_ok": {
|
|
"present": 5,
|
|
"ok": 4,
|
|
"fail": 1
|
|
}
|
|
},
|
|
"tier2_rates": {
|
|
"final_consistent_ok": 0.0,
|
|
"final_match_reported": 0.0
|
|
},
|
|
"tier2_counts": {
|
|
"final_consistent_ok": {
|
|
"present": 5,
|
|
"ok": 0,
|
|
"fail": 5
|
|
},
|
|
"final_match_reported": {
|
|
"present": 5,
|
|
"ok": 0,
|
|
"fail": 5
|
|
}
|
|
}
|
|
},
|
|
"stateful_followup": {
|
|
"n": 2,
|
|
"pass_rate": 1.0,
|
|
"pass_cols": [
|
|
"turn1_parse_ok",
|
|
"turn2_parse_ok",
|
|
"turn1_exact_match",
|
|
"turn2_exact_match"
|
|
],
|
|
"averages": {
|
|
"bucket_score": 2.0
|
|
},
|
|
"top_failures": [],
|
|
"signal_rates": {
|
|
"turn1_parse_ok": 1.0,
|
|
"turn2_parse_ok": 1.0,
|
|
"turn1_exact_match": 1.0,
|
|
"turn2_exact_match": 1.0
|
|
},
|
|
"signal_counts": {
|
|
"turn1_parse_ok": {
|
|
"present": 2,
|
|
"ok": 2,
|
|
"fail": 0
|
|
},
|
|
"turn2_parse_ok": {
|
|
"present": 2,
|
|
"ok": 2,
|
|
"fail": 0
|
|
},
|
|
"turn1_exact_match": {
|
|
"present": 2,
|
|
"ok": 2,
|
|
"fail": 0
|
|
},
|
|
"turn2_exact_match": {
|
|
"present": 2,
|
|
"ok": 2,
|
|
"fail": 0
|
|
}
|
|
},
|
|
"tier2_rates": {},
|
|
"tier2_counts": {}
|
|
},
|
|
"toolcall_only": {
|
|
"n": 2,
|
|
"pass_rate": 0.0,
|
|
"pass_cols": [
|
|
"tool_name_ok",
|
|
"args_ok"
|
|
],
|
|
"averages": {
|
|
"bucket_score": 1.0
|
|
},
|
|
"top_failures": [
|
|
{
|
|
"reason": "args_ok",
|
|
"count": 2
|
|
}
|
|
],
|
|
"signal_rates": {
|
|
"tool_name_ok": 1.0,
|
|
"args_ok": 0.0
|
|
},
|
|
"signal_counts": {
|
|
"tool_name_ok": {
|
|
"present": 2,
|
|
"ok": 2,
|
|
"fail": 0
|
|
},
|
|
"args_ok": {
|
|
"present": 2,
|
|
"ok": 0,
|
|
"fail": 2
|
|
}
|
|
},
|
|
"tier2_rates": {},
|
|
"tier2_counts": {}
|
|
},
|
|
"mixed_brief_json": {
|
|
"n": 2,
|
|
"pass_rate": 1.0,
|
|
"pass_cols": [
|
|
"answer_line_ok",
|
|
"json_parse_ok",
|
|
"schema_ok"
|
|
],
|
|
"averages": {
|
|
"bucket_score": 2.0
|
|
},
|
|
"top_failures": [],
|
|
"signal_rates": {
|
|
"answer_line_ok": 1.0,
|
|
"json_parse_ok": 1.0,
|
|
"schema_ok": 1.0
|
|
},
|
|
"signal_counts": {
|
|
"answer_line_ok": {
|
|
"present": 2,
|
|
"ok": 2,
|
|
"fail": 0
|
|
},
|
|
"json_parse_ok": {
|
|
"present": 2,
|
|
"ok": 2,
|
|
"fail": 0
|
|
},
|
|
"schema_ok": {
|
|
"present": 2,
|
|
"ok": 2,
|
|
"fail": 0
|
|
}
|
|
},
|
|
"tier2_rates": {},
|
|
"tier2_counts": {}
|
|
},
|
|
"json": {
|
|
"n": 4,
|
|
"pass_rate": null,
|
|
"pass_cols": [
|
|
"schema_ok",
|
|
"constraints_ok"
|
|
],
|
|
"averages": {
|
|
"bucket_score": 10.0
|
|
},
|
|
"top_failures": [],
|
|
"signal_rates": {},
|
|
"signal_counts": {},
|
|
"tier2_rates": {},
|
|
"tier2_counts": {}
|
|
},
|
|
"fuzz": {
|
|
"n": 20,
|
|
"pass_rate": null,
|
|
"pass_cols": [
|
|
"schema_ok",
|
|
"constraints_ok"
|
|
],
|
|
"averages": {
|
|
"bucket_score": 10.0
|
|
},
|
|
"top_failures": [],
|
|
"signal_rates": {},
|
|
"signal_counts": {},
|
|
"tier2_rates": {},
|
|
"tier2_counts": {}
|
|
},
|
|
"toolcall": {
|
|
"n": 2,
|
|
"pass_rate": 1.0,
|
|
"pass_cols": [
|
|
"stage1_tool_parse_ok",
|
|
"stage1_tool_schema_ok"
|
|
],
|
|
"averages": {
|
|
"bucket_score": 11.0
|
|
},
|
|
"top_failures": [],
|
|
"signal_rates": {
|
|
"stage1_tool_parse_ok": 1.0,
|
|
"stage1_tool_schema_ok": 1.0
|
|
},
|
|
"signal_counts": {
|
|
"stage1_tool_parse_ok": {
|
|
"present": 2,
|
|
"ok": 2,
|
|
"fail": 0
|
|
},
|
|
"stage1_tool_schema_ok": {
|
|
"present": 2,
|
|
"ok": 2,
|
|
"fail": 0
|
|
}
|
|
},
|
|
"tier2_rates": {},
|
|
"tier2_counts": {}
|
|
},
|
|
"mcq": {
|
|
"n": 5,
|
|
"pass_rate": null,
|
|
"pass_cols": [
|
|
"choice_ok"
|
|
],
|
|
"averages": {
|
|
"bucket_score": 1.0
|
|
},
|
|
"top_failures": [],
|
|
"signal_rates": {},
|
|
"signal_counts": {},
|
|
"tier2_rates": {},
|
|
"tier2_counts": {}
|
|
}
|
|
}
|
|
},
|
|
"generated_from": {
|
|
"rows": 42
|
|
},
|
|
"version_tag": "v7.21",
|
|
"run_id": "20260210_215029"
|
|
} |