339 lines
8.7 KiB
JSON
339 lines
8.7 KiB
JSON
{
|
|
"details": [
|
|
{
|
|
"difficulty": "hard",
|
|
"problem_id": "arc196_a",
|
|
"samples": [
|
|
{
|
|
"code_chars": 719,
|
|
"completion_chars": 828,
|
|
"end": "runtime_error",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"code_chars": 115,
|
|
"completion_chars": 2707,
|
|
"end": "wrong_answer",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"code_chars": 389,
|
|
"completion_chars": 1655,
|
|
"end": "timeout",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"code_chars": 214,
|
|
"completion_chars": 2386,
|
|
"end": "runtime_error",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"code_chars": 282,
|
|
"completion_chars": 2263,
|
|
"end": "runtime_error",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"code_chars": 303,
|
|
"completion_chars": 2072,
|
|
"end": "wrong_answer",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"difficulty": "hard",
|
|
"problem_id": "abc398_f",
|
|
"samples": [
|
|
{
|
|
"code_chars": 165,
|
|
"completion_chars": 3251,
|
|
"end": "wrong_answer",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"code_chars": 160,
|
|
"completion_chars": 3358,
|
|
"end": "wrong_answer",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"code_chars": 877,
|
|
"completion_chars": 985,
|
|
"end": "runtime_error",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"code_chars": 499,
|
|
"completion_chars": 2528,
|
|
"end": "wrong_answer",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"code_chars": 396,
|
|
"completion_chars": 2252,
|
|
"end": "wrong_answer",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"code_chars": 327,
|
|
"completion_chars": 2724,
|
|
"end": "runtime_error",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"difficulty": "easy",
|
|
"problem_id": "abc400_a",
|
|
"samples": [
|
|
{
|
|
"code_chars": 125,
|
|
"completion_chars": 12597,
|
|
"end": "wrong_answer",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"code_chars": 88,
|
|
"completion_chars": 12702,
|
|
"end": "wrong_answer",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"code_chars": 143,
|
|
"completion_chars": 12602,
|
|
"end": "wrong_answer",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"code_chars": 103,
|
|
"completion_chars": 12689,
|
|
"end": "wrong_answer",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"code_chars": 30,
|
|
"completion_chars": 12918,
|
|
"end": "wrong_answer",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"code_chars": 0,
|
|
"completion_chars": 2353,
|
|
"end": "format_error: Could not parse 'Tool type' or 'Tool query' from the response. Ensure you output an <answer> block with 'Tool type:' and 'Tool query:'.",
|
|
"passed": false,
|
|
"stop_seen": false,
|
|
"tool_type": null
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"difficulty": "hard",
|
|
"problem_id": "arc196_c",
|
|
"samples": [
|
|
{
|
|
"code_chars": 111,
|
|
"completion_chars": 3537,
|
|
"end": "runtime_error",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"code_chars": 231,
|
|
"completion_chars": 2460,
|
|
"end": "wrong_answer",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"code_chars": 233,
|
|
"completion_chars": 3048,
|
|
"end": "wrong_answer",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"code_chars": 132,
|
|
"completion_chars": 3325,
|
|
"end": "wrong_answer",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"code_chars": 870,
|
|
"completion_chars": 979,
|
|
"end": "wrong_answer",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"code_chars": 389,
|
|
"completion_chars": 2591,
|
|
"end": "wrong_answer",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"difficulty": "medium",
|
|
"problem_id": "abc400_d",
|
|
"samples": [
|
|
{
|
|
"code_chars": 733,
|
|
"completion_chars": 9798,
|
|
"end": "wrong_answer",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"code_chars": 0,
|
|
"completion_chars": 9689,
|
|
"end": "format_error: Could not parse 'Tool type' or 'Tool query' from the response. Ensure you output an <answer> block with 'Tool type:' and 'Tool query:'.",
|
|
"passed": false,
|
|
"stop_seen": false,
|
|
"tool_type": null
|
|
},
|
|
{
|
|
"code_chars": 1866,
|
|
"completion_chars": 7054,
|
|
"end": "runtime_error",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"code_chars": 1347,
|
|
"completion_chars": 8086,
|
|
"end": "runtime_error",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"code_chars": 0,
|
|
"completion_chars": 2217,
|
|
"end": "format_error: Could not parse 'Tool type' or 'Tool query' from the response. Ensure you output an <answer> block with 'Tool type:' and 'Tool query:'.",
|
|
"passed": false,
|
|
"stop_seen": false,
|
|
"tool_type": null
|
|
},
|
|
{
|
|
"code_chars": 1162,
|
|
"completion_chars": 8951,
|
|
"end": "runtime_error",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"difficulty": "hard",
|
|
"problem_id": "abc398_g",
|
|
"samples": [
|
|
{
|
|
"code_chars": 0,
|
|
"completion_chars": 4543,
|
|
"end": "format_error: Could not parse 'Tool type' or 'Tool query' from the response. Ensure you output an <answer> block with 'Tool type:' and 'Tool query:'.",
|
|
"passed": false,
|
|
"stop_seen": false,
|
|
"tool_type": null
|
|
},
|
|
{
|
|
"code_chars": 2840,
|
|
"completion_chars": 3882,
|
|
"end": "timeout",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"code_chars": 347,
|
|
"completion_chars": 11776,
|
|
"end": "wrong_answer",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"code_chars": 0,
|
|
"completion_chars": 5113,
|
|
"end": "format_error: Could not parse 'Tool type' or 'Tool query' from the response. Ensure you output an <answer> block with 'Tool type:' and 'Tool query:'.",
|
|
"passed": false,
|
|
"stop_seen": false,
|
|
"tool_type": null
|
|
},
|
|
{
|
|
"code_chars": 791,
|
|
"completion_chars": 11156,
|
|
"end": "runtime_error",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"code_chars": 1219,
|
|
"completion_chars": 10183,
|
|
"end": "runtime_error",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
}
|
|
]
|
|
}
|
|
],
|
|
"eval_limit": 6,
|
|
"max_new_tokens": 1024,
|
|
"pass_at_1": 0.0,
|
|
"passed_samples": 0,
|
|
"samples_per_problem": 6,
|
|
"tag": "final",
|
|
"temperature": 1.0,
|
|
"tool_turn_limit": 1,
|
|
"top_p": 0.95,
|
|
"total_samples": 36
|
|
}
|