917 lines
29 KiB
JSON
917 lines
29 KiB
JSON
{
|
|
"best": "stage1_group3",
|
|
"results": {
|
|
"after_sft": {
|
|
"details": [
|
|
{
|
|
"difficulty": "hard",
|
|
"problem_id": "arc196_a",
|
|
"samples": [
|
|
{
|
|
"completion_chars": 1731,
|
|
"end": "wrong_answer",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"completion_chars": 1841,
|
|
"end": "wrong_answer",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"completion_chars": 781,
|
|
"end": "runtime_error",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"completion_chars": 2276,
|
|
"end": "wrong_answer",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"completion_chars": 1543,
|
|
"end": "wrong_answer",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"completion_chars": 2677,
|
|
"end": "format_error: Could not parse 'Tool type' or 'Tool query' from the response. Ensure you output an <answer> block with 'Tool type:' and 'Tool query:'.",
|
|
"passed": false,
|
|
"stop_seen": false,
|
|
"tool_type": null
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"difficulty": "hard",
|
|
"problem_id": "abc398_f",
|
|
"samples": [
|
|
{
|
|
"completion_chars": 3067,
|
|
"end": "format_error: Could not parse 'Tool type' or 'Tool query' from the response. Ensure you output an <answer> block with 'Tool type:' and 'Tool query:'.",
|
|
"passed": false,
|
|
"stop_seen": false,
|
|
"tool_type": null
|
|
},
|
|
{
|
|
"completion_chars": 2838,
|
|
"end": "wrong_answer",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"completion_chars": 599,
|
|
"end": "format_error: Could not parse 'Tool type' or 'Tool query' from the response. Ensure you output an <answer> block with 'Tool type:' and 'Tool query:'.",
|
|
"passed": false,
|
|
"stop_seen": false,
|
|
"tool_type": null
|
|
},
|
|
{
|
|
"completion_chars": 2426,
|
|
"end": "wrong_answer",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"completion_chars": 1308,
|
|
"end": "runtime_error",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"completion_chars": 1743,
|
|
"end": "wrong_answer",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"difficulty": "easy",
|
|
"problem_id": "abc400_a",
|
|
"samples": [
|
|
{
|
|
"completion_chars": 376,
|
|
"end": "wrong_answer",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"completion_chars": 520,
|
|
"end": "wrong_answer",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"completion_chars": 725,
|
|
"end": "ok",
|
|
"passed": true,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"completion_chars": 666,
|
|
"end": "wrong_answer",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"completion_chars": 294,
|
|
"end": "format_error: Could not parse 'Tool type' or 'Tool query' from the response. Ensure you output an <answer> block with 'Tool type:' and 'Tool query:'.",
|
|
"passed": false,
|
|
"stop_seen": false,
|
|
"tool_type": null
|
|
},
|
|
{
|
|
"completion_chars": 431,
|
|
"end": "wrong_answer",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"difficulty": "hard",
|
|
"problem_id": "arc196_c",
|
|
"samples": [
|
|
{
|
|
"completion_chars": 10417,
|
|
"end": "runtime_error",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"completion_chars": 11700,
|
|
"end": "runtime_error",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"completion_chars": 1704,
|
|
"end": "format_error: Could not parse 'Tool type' or 'Tool query' from the response. Ensure you output an <answer> block with 'Tool type:' and 'Tool query:'.",
|
|
"passed": false,
|
|
"stop_seen": false,
|
|
"tool_type": null
|
|
},
|
|
{
|
|
"completion_chars": 3592,
|
|
"end": "format_error: Could not parse 'Tool type' or 'Tool query' from the response. Ensure you output an <answer> block with 'Tool type:' and 'Tool query:'.",
|
|
"passed": false,
|
|
"stop_seen": false,
|
|
"tool_type": null
|
|
},
|
|
{
|
|
"completion_chars": 12478,
|
|
"end": "runtime_error",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"completion_chars": 11832,
|
|
"end": "wrong_answer",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"difficulty": "medium",
|
|
"problem_id": "abc400_d",
|
|
"samples": [
|
|
{
|
|
"completion_chars": 5287,
|
|
"end": "runtime_error",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"completion_chars": 10882,
|
|
"end": "runtime_error",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"completion_chars": 1065,
|
|
"end": "format_error: Could not parse 'Tool type' or 'Tool query' from the response. Ensure you output an <answer> block with 'Tool type:' and 'Tool query:'.",
|
|
"passed": false,
|
|
"stop_seen": false,
|
|
"tool_type": null
|
|
},
|
|
{
|
|
"completion_chars": 1647,
|
|
"end": "format_error: Could not parse 'Tool type' or 'Tool query' from the response. Ensure you output an <answer> block with 'Tool type:' and 'Tool query:'.",
|
|
"passed": false,
|
|
"stop_seen": false,
|
|
"tool_type": null
|
|
},
|
|
{
|
|
"completion_chars": 13229,
|
|
"end": "format_error: Could not parse 'Tool type' or 'Tool query' from the response. Ensure you output an <answer> block with 'Tool type:' and 'Tool query:'.",
|
|
"passed": false,
|
|
"stop_seen": false,
|
|
"tool_type": null
|
|
},
|
|
{
|
|
"completion_chars": 9263,
|
|
"end": "timeout",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"difficulty": "hard",
|
|
"problem_id": "abc398_g",
|
|
"samples": [
|
|
{
|
|
"completion_chars": 12369,
|
|
"end": "wrong_answer",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"completion_chars": 10642,
|
|
"end": "runtime_error",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"completion_chars": 11360,
|
|
"end": "wrong_answer",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"completion_chars": 2075,
|
|
"end": "format_error: Could not parse 'Tool type' or 'Tool query' from the response. Ensure you output an <answer> block with 'Tool type:' and 'Tool query:'.",
|
|
"passed": false,
|
|
"stop_seen": false,
|
|
"tool_type": null
|
|
},
|
|
{
|
|
"completion_chars": 11910,
|
|
"end": "wrong_answer",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"completion_chars": 604,
|
|
"end": "format_error: Could not parse 'Tool type' or 'Tool query' from the response. Ensure you output an <answer> block with 'Tool type:' and 'Tool query:'.",
|
|
"passed": false,
|
|
"stop_seen": false,
|
|
"tool_type": null
|
|
}
|
|
]
|
|
}
|
|
],
|
|
"eval_limit": 6,
|
|
"max_new_tokens": 1024,
|
|
"pass_at_1": 0.027777777777777776,
|
|
"passed_samples": 1,
|
|
"samples_per_problem": 6,
|
|
"tag": "after_sft",
|
|
"temperature": 1.0,
|
|
"tool_turn_limit": 1,
|
|
"top_p": 0.95,
|
|
"total_samples": 36
|
|
},
|
|
"stage1_group3": {
|
|
"details": [
|
|
{
|
|
"difficulty": "hard",
|
|
"problem_id": "arc196_a",
|
|
"samples": [
|
|
{
|
|
"completion_chars": 11690,
|
|
"end": "wrong_answer",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"completion_chars": 12444,
|
|
"end": "wrong_answer",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"completion_chars": 12570,
|
|
"end": "wrong_answer",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"completion_chars": 12044,
|
|
"end": "format_error: Could not parse 'Tool type' or 'Tool query' from the response. Ensure you output an <answer> block with 'Tool type:' and 'Tool query:'.",
|
|
"passed": false,
|
|
"stop_seen": false,
|
|
"tool_type": null
|
|
},
|
|
{
|
|
"completion_chars": 1151,
|
|
"end": "format_error: Could not parse 'Tool type' or 'Tool query' from the response. Ensure you output an <answer> block with 'Tool type:' and 'Tool query:'.",
|
|
"passed": false,
|
|
"stop_seen": false,
|
|
"tool_type": null
|
|
},
|
|
{
|
|
"completion_chars": 10895,
|
|
"end": "format_error: Could not parse 'Tool type' or 'Tool query' from the response. Ensure you output an <answer> block with 'Tool type:' and 'Tool query:'.",
|
|
"passed": false,
|
|
"stop_seen": false,
|
|
"tool_type": null
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"difficulty": "hard",
|
|
"problem_id": "abc398_f",
|
|
"samples": [
|
|
{
|
|
"completion_chars": 2877,
|
|
"end": "wrong_answer",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"completion_chars": 2932,
|
|
"end": "format_error: Could not parse 'Tool type' or 'Tool query' from the response. Ensure you output an <answer> block with 'Tool type:' and 'Tool query:'.",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": null
|
|
},
|
|
{
|
|
"completion_chars": 715,
|
|
"end": "runtime_error",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"completion_chars": 2867,
|
|
"end": "wrong_answer",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"completion_chars": 2538,
|
|
"end": "wrong_answer",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"completion_chars": 2577,
|
|
"end": "wrong_answer",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"difficulty": "easy",
|
|
"problem_id": "abc400_a",
|
|
"samples": [
|
|
{
|
|
"completion_chars": 328,
|
|
"end": "runtime_error",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"completion_chars": 917,
|
|
"end": "wrong_answer",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"completion_chars": 600,
|
|
"end": "wrong_answer",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"completion_chars": 742,
|
|
"end": "ok",
|
|
"passed": true,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"completion_chars": 808,
|
|
"end": "wrong_answer",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"completion_chars": 820,
|
|
"end": "ok",
|
|
"passed": true,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"difficulty": "hard",
|
|
"problem_id": "arc196_c",
|
|
"samples": [
|
|
{
|
|
"completion_chars": 5294,
|
|
"end": "runtime_error",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"completion_chars": 1689,
|
|
"end": "runtime_error",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"completion_chars": 5976,
|
|
"end": "format_error: final",
|
|
"passed": false,
|
|
"stop_seen": false,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"completion_chars": 4300,
|
|
"end": "wrong_answer",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"completion_chars": 4040,
|
|
"end": "wrong_answer",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"completion_chars": 5089,
|
|
"end": "wrong_answer",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"difficulty": "medium",
|
|
"problem_id": "abc400_d",
|
|
"samples": [
|
|
{
|
|
"completion_chars": 8429,
|
|
"end": "wrong_answer",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"completion_chars": 7583,
|
|
"end": "runtime_error",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"completion_chars": 7201,
|
|
"end": "wrong_answer",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"completion_chars": 7008,
|
|
"end": "runtime_error",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"completion_chars": 2737,
|
|
"end": "format_error: final",
|
|
"passed": false,
|
|
"stop_seen": false,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"completion_chars": 7069,
|
|
"end": "runtime_error",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"difficulty": "hard",
|
|
"problem_id": "abc398_g",
|
|
"samples": [
|
|
{
|
|
"completion_chars": 11630,
|
|
"end": "wrong_answer",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"completion_chars": 12637,
|
|
"end": "wrong_answer",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"completion_chars": 4093,
|
|
"end": "format_error: Could not parse 'Tool type' or 'Tool query' from the response. Ensure you output an <answer> block with 'Tool type:' and 'Tool query:'.",
|
|
"passed": false,
|
|
"stop_seen": false,
|
|
"tool_type": null
|
|
},
|
|
{
|
|
"completion_chars": 3088,
|
|
"end": "format_error: Could not parse 'Tool type' or 'Tool query' from the response. Ensure you output an <answer> block with 'Tool type:' and 'Tool query:'.",
|
|
"passed": false,
|
|
"stop_seen": false,
|
|
"tool_type": null
|
|
},
|
|
{
|
|
"completion_chars": 13122,
|
|
"end": "format_error: Could not parse 'Tool type' or 'Tool query' from the response. Ensure you output an <answer> block with 'Tool type:' and 'Tool query:'.",
|
|
"passed": false,
|
|
"stop_seen": false,
|
|
"tool_type": null
|
|
},
|
|
{
|
|
"completion_chars": 2073,
|
|
"end": "format_error: Could not parse 'Tool type' or 'Tool query' from the response. Ensure you output an <answer> block with 'Tool type:' and 'Tool query:'.",
|
|
"passed": false,
|
|
"stop_seen": false,
|
|
"tool_type": null
|
|
}
|
|
]
|
|
}
|
|
],
|
|
"eval_limit": 6,
|
|
"max_new_tokens": 1024,
|
|
"pass_at_1": 0.05555555555555555,
|
|
"passed_samples": 2,
|
|
"samples_per_problem": 6,
|
|
"tag": "stage1_group3",
|
|
"temperature": 1.0,
|
|
"tool_turn_limit": 1,
|
|
"top_p": 0.95,
|
|
"total_samples": 36
|
|
},
|
|
"stage2_group1": {
|
|
"details": [
|
|
{
|
|
"difficulty": "hard",
|
|
"problem_id": "arc196_a",
|
|
"samples": [
|
|
{
|
|
"completion_chars": 11873,
|
|
"end": "wrong_answer",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"completion_chars": 4103,
|
|
"end": "format_error: Could not parse 'Tool type' or 'Tool query' from the response. Ensure you output an <answer> block with 'Tool type:' and 'Tool query:'.",
|
|
"passed": false,
|
|
"stop_seen": false,
|
|
"tool_type": null
|
|
},
|
|
{
|
|
"completion_chars": 10118,
|
|
"end": "format_error: Could not parse 'Tool type' or 'Tool query' from the response. Ensure you output an <answer> block with 'Tool type:' and 'Tool query:'.",
|
|
"passed": false,
|
|
"stop_seen": false,
|
|
"tool_type": null
|
|
},
|
|
{
|
|
"completion_chars": 11800,
|
|
"end": "wrong_answer",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"completion_chars": 2309,
|
|
"end": "format_error: Could not parse 'Tool type' or 'Tool query' from the response. Ensure you output an <answer> block with 'Tool type:' and 'Tool query:'.",
|
|
"passed": false,
|
|
"stop_seen": false,
|
|
"tool_type": null
|
|
},
|
|
{
|
|
"completion_chars": 2870,
|
|
"end": "format_error: final",
|
|
"passed": false,
|
|
"stop_seen": false,
|
|
"tool_type": "final"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"difficulty": "hard",
|
|
"problem_id": "abc398_f",
|
|
"samples": [
|
|
{
|
|
"completion_chars": 1513,
|
|
"end": "wrong_answer",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"completion_chars": 1075,
|
|
"end": "wrong_answer",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"completion_chars": 1550,
|
|
"end": "wrong_answer",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"completion_chars": 1186,
|
|
"end": "wrong_answer",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"completion_chars": 1775,
|
|
"end": "format_error: Could not parse 'Tool type' or 'Tool query' from the response. Ensure you output an <answer> block with 'Tool type:' and 'Tool query:'.",
|
|
"passed": false,
|
|
"stop_seen": false,
|
|
"tool_type": null
|
|
},
|
|
{
|
|
"completion_chars": 399,
|
|
"end": "wrong_answer",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"difficulty": "easy",
|
|
"problem_id": "abc400_a",
|
|
"samples": [
|
|
{
|
|
"completion_chars": 478,
|
|
"end": "format_error: Could not parse 'Tool type' or 'Tool query' from the response. Ensure you output an <answer> block with 'Tool type:' and 'Tool query:'.",
|
|
"passed": false,
|
|
"stop_seen": false,
|
|
"tool_type": null
|
|
},
|
|
{
|
|
"completion_chars": 466,
|
|
"end": "wrong_answer",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"completion_chars": 949,
|
|
"end": "format_error: Could not parse 'Tool type' or 'Tool query' from the response. Ensure you output an <answer> block with 'Tool type:' and 'Tool query:'.",
|
|
"passed": false,
|
|
"stop_seen": false,
|
|
"tool_type": null
|
|
},
|
|
{
|
|
"completion_chars": 556,
|
|
"end": "ok",
|
|
"passed": true,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"completion_chars": 328,
|
|
"end": "wrong_answer",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"completion_chars": 434,
|
|
"end": "wrong_answer",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"difficulty": "hard",
|
|
"problem_id": "arc196_c",
|
|
"samples": [
|
|
{
|
|
"completion_chars": 5189,
|
|
"end": "wrong_answer",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"completion_chars": 4221,
|
|
"end": "wrong_answer",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"completion_chars": 3838,
|
|
"end": "wrong_answer",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"completion_chars": 1365,
|
|
"end": "format_error: final",
|
|
"passed": false,
|
|
"stop_seen": false,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"completion_chars": 4021,
|
|
"end": "wrong_answer",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"completion_chars": 4585,
|
|
"end": "wrong_answer",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"difficulty": "medium",
|
|
"problem_id": "abc400_d",
|
|
"samples": [
|
|
{
|
|
"completion_chars": 1449,
|
|
"end": "format_error: Could not parse 'Tool type' or 'Tool query' from the response. Ensure you output an <answer> block with 'Tool type:' and 'Tool query:'.",
|
|
"passed": false,
|
|
"stop_seen": false,
|
|
"tool_type": null
|
|
},
|
|
{
|
|
"completion_chars": 13228,
|
|
"end": "format_error: Could not parse 'Tool type' or 'Tool query' from the response. Ensure you output an <answer> block with 'Tool type:' and 'Tool query:'.",
|
|
"passed": false,
|
|
"stop_seen": false,
|
|
"tool_type": null
|
|
},
|
|
{
|
|
"completion_chars": 7164,
|
|
"end": "runtime_error",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"completion_chars": 10712,
|
|
"end": "runtime_error",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"completion_chars": 9464,
|
|
"end": "wrong_answer",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"completion_chars": 5995,
|
|
"end": "runtime_error",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"difficulty": "hard",
|
|
"problem_id": "abc398_g",
|
|
"samples": [
|
|
{
|
|
"completion_chars": 10181,
|
|
"end": "runtime_error",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"completion_chars": 11037,
|
|
"end": "wrong_answer",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"completion_chars": 2580,
|
|
"end": "format_error: Could not parse 'Tool type' or 'Tool query' from the response. Ensure you output an <answer> block with 'Tool type:' and 'Tool query:'.",
|
|
"passed": false,
|
|
"stop_seen": false,
|
|
"tool_type": null
|
|
},
|
|
{
|
|
"completion_chars": 10968,
|
|
"end": "wrong_answer",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"completion_chars": 12204,
|
|
"end": "wrong_answer",
|
|
"passed": false,
|
|
"stop_seen": true,
|
|
"tool_type": "final"
|
|
},
|
|
{
|
|
"completion_chars": 2073,
|
|
"end": "format_error: Could not parse 'Tool type' or 'Tool query' from the response. Ensure you output an <answer> block with 'Tool type:' and 'Tool query:'.",
|
|
"passed": false,
|
|
"stop_seen": false,
|
|
"tool_type": null
|
|
}
|
|
]
|
|
}
|
|
],
|
|
"eval_limit": 6,
|
|
"max_new_tokens": 1024,
|
|
"pass_at_1": 0.027777777777777776,
|
|
"passed_samples": 1,
|
|
"samples_per_problem": 6,
|
|
"tag": "stage2_group1",
|
|
"temperature": 1.0,
|
|
"tool_turn_limit": 1,
|
|
"top_p": 0.95,
|
|
"total_samples": 36
|
|
}
|
|
},
|
|
"summary": {
|
|
"after_sft": 0.027777777777777776,
|
|
"stage1_group3": 0.05555555555555555,
|
|
"stage2_group1": 0.027777777777777776
|
|
}
|
|
}
|