339 lines
8.6 KiB
JSON
339 lines
8.6 KiB
JSON
{
|
|
"details": [
|
|
{
|
|
"difficulty": "hard",
|
|
"problem_id": "arc196_a",
|
|
"samples": [
|
|
{
|
|
"code_chars": 380,
|
|
"completion_chars": 2880,
|
|
"end": "wrong_answer",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"code_chars": 327,
|
|
"completion_chars": 4333,
|
|
"end": "runtime_error",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"code_chars": 0,
|
|
"completion_chars": 5950,
|
|
"end": "format_error: final",
|
|
"passed": false,
|
|
"stop_seen": false,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"code_chars": 318,
|
|
"completion_chars": 5576,
|
|
"end": "wrong_answer",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"code_chars": 316,
|
|
"completion_chars": 6824,
|
|
"end": "wrong_answer",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"code_chars": 540,
|
|
"completion_chars": 3408,
|
|
"end": "runtime_error",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"difficulty": "hard",
|
|
"problem_id": "abc398_f",
|
|
"samples": [
|
|
{
|
|
"code_chars": 332,
|
|
"completion_chars": 5783,
|
|
"end": "wrong_answer",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"code_chars": 325,
|
|
"completion_chars": 4684,
|
|
"end": "wrong_answer",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"code_chars": 351,
|
|
"completion_chars": 2437,
|
|
"end": "wrong_answer",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"code_chars": 194,
|
|
"completion_chars": 4745,
|
|
"end": "runtime_error",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"code_chars": 360,
|
|
"completion_chars": 3907,
|
|
"end": "ok",
|
|
"passed": true,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"code_chars": 0,
|
|
"completion_chars": 6143,
|
|
"end": "format_error: Could not parse 'Tool type' or 'Tool query' from the response. Ensure you output an <answer> block with 'Tool type:' and 'Tool query:'.",
|
|
"passed": false,
|
|
"stop_seen": false,
|
|
"tool_type": null
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"difficulty": "easy",
|
|
"problem_id": "abc400_a",
|
|
"samples": [
|
|
{
|
|
"code_chars": 180,
|
|
"completion_chars": 982,
|
|
"end": "ok",
|
|
"passed": true,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"code_chars": 0,
|
|
"completion_chars": 1062,
|
|
"end": "format_error: final",
|
|
"passed": false,
|
|
"stop_seen": false,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"code_chars": 285,
|
|
"completion_chars": 1636,
|
|
"end": "runtime_error",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"code_chars": 264,
|
|
"completion_chars": 1539,
|
|
"end": "runtime_error",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"code_chars": 165,
|
|
"completion_chars": 886,
|
|
"end": "ok",
|
|
"passed": true,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"code_chars": 175,
|
|
"completion_chars": 940,
|
|
"end": "ok",
|
|
"passed": true,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"difficulty": "hard",
|
|
"problem_id": "arc196_c",
|
|
"samples": [
|
|
{
|
|
"code_chars": 0,
|
|
"completion_chars": 9181,
|
|
"end": "format_error: Could not parse 'Tool type' or 'Tool query' from the response. Ensure you output an <answer> block with 'Tool type:' and 'Tool query:'.",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": null
|
|
},
|
|
{
|
|
"code_chars": 601,
|
|
"completion_chars": 7205,
|
|
"end": "wrong_answer",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"code_chars": 371,
|
|
"completion_chars": 3946,
|
|
"end": "runtime_error",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"code_chars": 1489,
|
|
"completion_chars": 8048,
|
|
"end": "runtime_error",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"code_chars": 0,
|
|
"completion_chars": 5614,
|
|
"end": "format_error: Could not parse 'Tool type' or 'Tool query' from the response. Ensure you output an <answer> block with 'Tool type:' and 'Tool query:'.",
|
|
"passed": false,
|
|
"stop_seen": false,
|
|
"tool_type": null
|
|
},
|
|
{
|
|
"code_chars": 125,
|
|
"completion_chars": 7651,
|
|
"end": "wrong_answer",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"difficulty": "medium",
|
|
"problem_id": "abc400_d",
|
|
"samples": [
|
|
{
|
|
"code_chars": 2468,
|
|
"completion_chars": 5103,
|
|
"end": "timeout",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"code_chars": 0,
|
|
"completion_chars": 3814,
|
|
"end": "format_error: final",
|
|
"passed": false,
|
|
"stop_seen": false,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"code_chars": 1121,
|
|
"completion_chars": 9244,
|
|
"end": "wrong_answer",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"code_chars": 0,
|
|
"completion_chars": 3572,
|
|
"end": "format_error: final",
|
|
"passed": false,
|
|
"stop_seen": false,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"code_chars": 1874,
|
|
"completion_chars": 4560,
|
|
"end": "wrong_answer",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"code_chars": 0,
|
|
"completion_chars": 3678,
|
|
"end": "format_error: final",
|
|
"passed": false,
|
|
"stop_seen": false,
|
|
"tool_type": "final"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"difficulty": "hard",
|
|
"problem_id": "abc398_g",
|
|
"samples": [
|
|
{
|
|
"code_chars": 820,
|
|
"completion_chars": 3243,
|
|
"end": "runtime_error",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"code_chars": 1127,
|
|
"completion_chars": 5958,
|
|
"end": "runtime_error",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"code_chars": 842,
|
|
"completion_chars": 6255,
|
|
"end": "wrong_answer",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"code_chars": 326,
|
|
"completion_chars": 6262,
|
|
"end": "wrong_answer",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"code_chars": 869,
|
|
"completion_chars": 4210,
|
|
"end": "runtime_error",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"code_chars": 0,
|
|
"completion_chars": 3812,
|
|
"end": "format_error: Could not parse 'Tool type' or 'Tool query' from the response. Ensure you output an <answer> block with 'Tool type:' and 'Tool query:'.",
|
|
"passed": false,
|
|
"stop_seen": false,
|
|
"tool_type": null
|
|
}
|
|
]
|
|
}
|
|
],
|
|
"eval_limit": 6,
|
|
"max_new_tokens": 1024,
|
|
"pass_at_1": 0.1111111111111111,
|
|
"passed_samples": 4,
|
|
"samples_per_problem": 6,
|
|
"tag": "baseline",
|
|
"temperature": 1.0,
|
|
"tool_turn_limit": 1,
|
|
"top_p": 0.95,
|
|
"total_samples": 36
|
|
}
|