bk1dr/qwen3-8b-code-pkpo
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

初始化项目，由ModelHub XC社区提供模型

Browse Source 
Model: bk1dr/qwen3-8b-code-pkpo
Source: Original Platform
This commit is contained in:
ModelHub XC
2026-07-30 09:33:20 +08:00
commit 8060a07a88
29 changed files with 158812 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

338
eval/baseline_results.json Normal file
View File

@@ -0,0 +1,338 @@
{
"details": [
{
"difficulty": "hard",
"problem_id": "arc196_a",
"samples": [
{
"code_chars": 380,
"completion_chars": 2880,
"end": "wrong_answer",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"code_chars": 327,
"completion_chars": 4333,
"end": "runtime_error",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"code_chars": 0,
"completion_chars": 5950,
"end": "format_error: final",
"passed": false,
"stop_seen": false,
"tool_type": "final"
},
{
"code_chars": 318,
"completion_chars": 5576,
"end": "wrong_answer",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"code_chars": 316,
"completion_chars": 6824,
"end": "wrong_answer",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"code_chars": 540,
"completion_chars": 3408,
"end": "runtime_error",
"passed": false,
"stop_seen": true,
"tool_type": "final"
}
]
},
{
"difficulty": "hard",
"problem_id": "abc398_f",
"samples": [
{
"code_chars": 332,
"completion_chars": 5783,
"end": "wrong_answer",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"code_chars": 325,
"completion_chars": 4684,
"end": "wrong_answer",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"code_chars": 351,
"completion_chars": 2437,
"end": "wrong_answer",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"code_chars": 194,
"completion_chars": 4745,
"end": "runtime_error",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"code_chars": 360,
"completion_chars": 3907,
"end": "ok",
"passed": true,
"stop_seen": true,
"tool_type": "final"
},
{
"code_chars": 0,
"completion_chars": 6143,
"end": "format_error: Could not parse 'Tool type' or 'Tool query' from the response. Ensure you output an <answer> block with 'Tool type:' and 'Tool query:'.",
"passed": false,
"stop_seen": false,
"tool_type": null
}
]
},
{
"difficulty": "easy",
"problem_id": "abc400_a",
"samples": [
{
"code_chars": 180,
"completion_chars": 982,
"end": "ok",
"passed": true,
"stop_seen": true,
"tool_type": "final"
},
{
"code_chars": 0,
"completion_chars": 1062,
"end": "format_error: final",
"passed": false,
"stop_seen": false,
"tool_type": "final"
},
{
"code_chars": 285,
"completion_chars": 1636,
"end": "runtime_error",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"code_chars": 264,
"completion_chars": 1539,
"end": "runtime_error",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"code_chars": 165,
"completion_chars": 886,
"end": "ok",
"passed": true,
"stop_seen": true,
"tool_type": "final"
},
{
"code_chars": 175,
"completion_chars": 940,
"end": "ok",
"passed": true,
"stop_seen": true,
"tool_type": "final"
}
]
},
{
"difficulty": "hard",
"problem_id": "arc196_c",
"samples": [
{
"code_chars": 0,
"completion_chars": 9181,
"end": "format_error: Could not parse 'Tool type' or 'Tool query' from the response. Ensure you output an <answer> block with 'Tool type:' and 'Tool query:'.",
"passed": false,
"stop_seen": true,
"tool_type": null
},
{
"code_chars": 601,
"completion_chars": 7205,
"end": "wrong_answer",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"code_chars": 371,
"completion_chars": 3946,
"end": "runtime_error",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"code_chars": 1489,
"completion_chars": 8048,
"end": "runtime_error",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"code_chars": 0,
"completion_chars": 5614,
"end": "format_error: Could not parse 'Tool type' or 'Tool query' from the response. Ensure you output an <answer> block with 'Tool type:' and 'Tool query:'.",
"passed": false,
"stop_seen": false,
"tool_type": null
},
{
"code_chars": 125,
"completion_chars": 7651,
"end": "wrong_answer",
"passed": false,
"stop_seen": true,
"tool_type": "final"
}
]
},
{
"difficulty": "medium",
"problem_id": "abc400_d",
"samples": [
{
"code_chars": 2468,
"completion_chars": 5103,
"end": "timeout",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"code_chars": 0,
"completion_chars": 3814,
"end": "format_error: final",
"passed": false,
"stop_seen": false,
"tool_type": "final"
},
{
"code_chars": 1121,
"completion_chars": 9244,
"end": "wrong_answer",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"code_chars": 0,
"completion_chars": 3572,
"end": "format_error: final",
"passed": false,
"stop_seen": false,
"tool_type": "final"
},
{
"code_chars": 1874,
"completion_chars": 4560,
"end": "wrong_answer",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"code_chars": 0,
"completion_chars": 3678,
"end": "format_error: final",
"passed": false,
"stop_seen": false,
"tool_type": "final"
}
]
},
{
"difficulty": "hard",
"problem_id": "abc398_g",
"samples": [
{
"code_chars": 820,
"completion_chars": 3243,
"end": "runtime_error",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"code_chars": 1127,
"completion_chars": 5958,
"end": "runtime_error",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"code_chars": 842,
"completion_chars": 6255,
"end": "wrong_answer",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"code_chars": 326,
"completion_chars": 6262,
"end": "wrong_answer",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"code_chars": 869,
"completion_chars": 4210,
"end": "runtime_error",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"code_chars": 0,
"completion_chars": 3812,
"end": "format_error: Could not parse 'Tool type' or 'Tool query' from the response. Ensure you output an <answer> block with 'Tool type:' and 'Tool query:'.",
"passed": false,
"stop_seen": false,
"tool_type": null
}
]
}
],
"eval_limit": 6,
"max_new_tokens": 1024,
"pass_at_1": 0.1111111111111111,
"passed_samples": 4,
"samples_per_problem": 6,
"tag": "baseline",
"temperature": 1.0,
"tool_turn_limit": 1,
"top_p": 0.95,
"total_samples": 36
}

247
eval/eval_subset.json Normal file
View File

@@ -0,0 +1,247 @@
[
{
"contest_date": "2025-04-06T00:00:00",
"difficulty": "hard",
"hidden_tests": [
{
"input": "4564\n253140288 293305612 750791934 354530934 444237786 846682985 925663544 35056",
"output_len": 14
},
{
"input": "63\n55449 10966 48278 28854 34199 76708 21835 56522 25159 46992 15075 8370 92014 ",
"output_len": 8
},
{
"input": "52\n68827 19447 33671 78983 20435 49677 76409 38586 94127 92586 61661 8698 11095 ",
"output_len": 8
}
],
"problem_id": "arc196_a",
"public_tests": [
{
"input": "4\n1 2 5 3",
"output": "5",
"testtype": "stdin"
},
{
"input": "7\n3 1 4 1 5 9 2",
"output": "14",
"testtype": "stdin"
},
{
"input": "5\n1 1 1 1 1",
"output": "0",
"testtype": "stdin"
}
],
"source": "livecodebench/code_generation_lite:v6",
"starter_code": "",
"statement": "You are given a length-N sequence A = (A_1, A_2, \\ldots, A_N).\nYou will repeatedly perform the following operation until the sequence has length at most 1: choose two adjacent numbers and remove both from the sequence.\nThe score obtained in one operation is the absolute difference of the two chosen numbers.\nFind the maximum possible total score obtained.\n\nInput\n\nThe input is given from Standard Input in the following format:\nN\nA_1 A_2 \\ldots A_N\n\nOutput\n\nPrint the maximum possible total score ob",
"title": "Adjacent Delete"
},
{
"contest_date": "2025-03-22T00:00:00",
"difficulty": "hard",
"hidden_tests": [
{
"input": "ZGBZMOLYGOLZUCBBPIAQVSSGHCYUYQWQNQJDENSNCDNCDNYEXAZONVNAPKXICLCDLWALLFAHLCTEGAGV",
"output_len": 166
},
{
"input": "UOMWVSNBMWSNYVWBFOCIWFOQPRTYABPKJOBZZNGRUCXEAMVNKAGAWYAVQTDGDTUG",
"output_len": 128
},
{
"input": "XCU",
"output_len": 6
}
],
"problem_id": "abc398_f",
"public_tests": [
{
"input": "ABC",
"output": "ABCBA",
"testtype": "stdin"
},
{
"input": "Z",
"output": "Z",
"testtype": "stdin"
},
{
"input": "TREE",
"output": "TREERT",
"testtype": "stdin"
}
],
"source": "livecodebench/code_generation_lite:v6",
"starter_code": "",
"statement": "Find one shortest palindrome that has S as its prefix.\n\nInput\n\nThe input is given from Standard Input in the following format:\nS\n\nOutput\n\nPrint the answer.\r\nIf multiple solutions exist, any of them is accepted.\n\nConstraints\n\n\n- S is a string of length between 1 and 500000, inclusive, consisting of uppercase English letters.\n\nSample Input 1\n\nABC\n\nSample Output 1\n\nABCBA\r\n\nABCBA is a shortest palindrome that has S= ABC as its prefix.\n\nSample Input 2\n\nZ\n\nSample Output 2\n\nZ\r\n\nZ is a shortest palindro",
"title": "ABCBA"
},
{
"contest_date": "2025-04-05T00:00:00",
"difficulty": "easy",
"hidden_tests": [
{
"input": "286",
"output_len": 3
},
{
"input": "20",
"output_len": 3
},
{
"input": "26",
"output_len": 3
}
],
"problem_id": "abc400_a",
"public_tests": [
{
"input": "10",
"output": "40",
"testtype": "stdin"
},
{
"input": "11",
"output": "-1",
"testtype": "stdin"
},
{
"input": "400",
"output": "1",
"testtype": "stdin"
}
],
"source": "livecodebench/code_generation_lite:v6",
"starter_code": "",
"statement": "In the ceremony commemorating ABC400, we want to arrange 400 people in a rectangular formation of A rows and B columns without any gaps.\nYou are given a positive integer A. Print the value of a positive integer B for which such an arrangement is possible. If there is no such positive integer B, print -1.\n\nInput\n\nThe input is given from Standard Input in the following format:\nA\n\nOutput\n\nPrint the value of B or -1 as specified by the problem statement.\n\nConstraints\n\n\n- A is an integer between 1 an",
"title": "ABC400 Party"
},
{
"contest_date": "2025-04-06T00:00:00",
"difficulty": "hard",
"hidden_tests": [
{
"input": "118\nWWWWBWBWWBWBWWWWWWWBWBWWBBBBBBBWBBWBBWBWBWBWBWWWBBBWBWBWBWWBWWWWWWBBWWBBWBWB",
"output_len": 2
},
{
"input": "108\nBBBWWBWBWWBWBBWWBWWBWBWWBWWWWBWWWBWBWBWWWBBWWWBBWWWBWBBBWWWWBBWBWBBBWBWWWBWB",
"output_len": 2
},
{
"input": "8\nWWBWBWWBWBBBWWBB",
"output_len": 2
}
],
"problem_id": "arc196_c",
"public_tests": [
{
"input": "2\nBWBW",
"output": "1",
"testtype": "stdin"
},
{
"input": "4\nBWWBWBWB",
"output": "0",
"testtype": "stdin"
},
{
"input": "9\nBWWBWBBBWWBWBBWWBW",
"output": "240792",
"testtype": "stdin"
}
],
"source": "livecodebench/code_generation_lite:v6",
"starter_code": "",
"statement": "There is a directed graph with 2N vertices and 2N-1 edges.\r\nThe vertices are numbered 1, 2, \\ldots, 2N, and the i-th edge is a directed edge from vertex i to vertex i+1.\nYou are given a length-2N string S = S_1 S_2 \\ldots S_{2N} consisting of N Ws and N Bs.\r\nVertex i is colored white if S_i is W, and black if S_i is B.\nYou will perform the following series of operations:\n\n- Partition the 2N vertices into N pairs, each consisting of one white vertex and one black vertex.\n- For each pair, add a di",
"title": "Strongly Connected"
},
{
"contest_date": "2025-04-05T00:00:00",
"difficulty": "medium",
"hidden_tests": [
{
"input": "2 2\n..\n..\n2 2 2 1",
"output_len": 2
},
{
"input": "8 5\n....#\n...#.\n.##.#\n###.#\n##...\n###.#\n.#..#\n.##..\n4 4 2 5",
"output_len": 2
},
{
"input": "6 4\n..##\n...#\n##..\n.#..\n..#.\n...#\n4 1 1 1",
"output_len": 2
}
],
"problem_id": "abc400_d",
"public_tests": [
{
"input": "10 10\n..........\n#########.\n#.......#.\n#..####.#.\n##....#.#.\n#####.#.#.\n.##.#.#.#.\n###.#.#.#.\n###.#.#.#.\n#.....#...\n1 1 7 1",
"output": "1",
"testtype": "stdin"
},
{
"input": "2 2\n.#\n#.\n1 1 2 2",
"output": "1",
"testtype": "stdin"
},
{
"input": "1 3\n.#.\n1 1 1 3",
"output": "1",
"testtype": "stdin"
},
{
"input": "20 20\n####################\n##...##....###...###\n#.....#.....#.....##\n#..#..#..#..#..#..##\n#..#..#....##..#####\n#.....#.....#..#####\n#.....#..#..#..#..##\n#..#..#.....#.....##\n#..#..#....###...###\n####################\n####################\n##..#..##...###...##\n##..#..#.....#.....#\n##..#..#..#..#..#..#\n##..#..#..#..#..#..#\n##.....#..#..#..#..#\n###....#..#..#..#..#\n#####..#.....#.....#\n#####..##...###...##\n####################\n3 3 18 18",
"output": "3",
"testtype": "stdin"
}
],
"source": "livecodebench/code_generation_lite:v6",
"starter_code": "",
"statement": "Takahashi is about to go buy eel at a fish shop.\nThe town where he lives is divided into a grid of H rows and W columns. Each cell is either a road or a wall.\r\nLet us denote the cell at the i-th row from the top (1\\leq i \\leq H) and the j-th column from the left (1\\leq j \\leq W) as cell (i,j).\r\nInformation about each cell is given by H strings S_1,S_2,\\ldots,S_H, each of length W. Specifically, if the j-th character of S_i (1\\leq i \\leq H,1\\leq j\\leq W) is ., cell (i,j) is a road; if it is #, ce",
"title": "Takahashi the Wall Breaker"
},
{
"contest_date": "2025-03-22T00:00:00",
"difficulty": "hard",
"hidden_tests": [
{
"input": "5 4\n1 2\n1 3\n1 4\n1 5",
"output_len": 10
},
{
"input": "128 2871\n1 68\n24 62\n13 107\n21 41\n6 89\n10 106\n24 88\n25 53\n21 44\n2 72\n2 95\n29 79\n1",
"output_len": 10
},
{
"input": "6 5\n1 6\n3 6\n2 6\n4 6\n5 6",
"output_len": 10
}
],
"problem_id": "abc398_g",
"public_tests": [
{
"input": "4 3\n1 2\n2 3\n3 4",
"output": "Aoki",
"testtype": "stdin"
},
{
"input": "4 2\n1 2\n3 4",
"output": "Takahashi",
"testtype": "stdin"
},
{
"input": "9 5\n2 9\n2 3\n4 6\n5 7\n1 8",
"output": "Aoki",
"testtype": "stdin"
}
],
"source": "livecodebench/code_generation_lite:v6",
"starter_code": "",
"statement": "You are given a simple undirected graph with N vertices and M edges, with vertices labeled 1 to N and edges labeled 1 to M. The i-th edge connects vertices U_i and V_i. Initially, G does not contain an odd cycle.\nTakahashi and Aoki will play a game using this graph G. With Aoki going first, they take turns performing the following operation:\n\n- Choose a pair of integers (i,j) with 1 \\leq i < j \\leq N that satisfies both of the following conditions, then add an edge connecting vertices i and j to",
"title": "Not Only Tree Game"
}
]

302
eval/final_results.json Normal file
View File

@@ -0,0 +1,302 @@
{
"details": [
{
"difficulty": "hard",
"problem_id": "arc196_a",
"samples": [
{
"completion_chars": 11690,
"end": "wrong_answer",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"completion_chars": 12444,
"end": "wrong_answer",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"completion_chars": 12570,
"end": "wrong_answer",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"completion_chars": 12044,
"end": "format_error: Could not parse 'Tool type' or 'Tool query' from the response. Ensure you output an <answer> block with 'Tool type:' and 'Tool query:'.",
"passed": false,
"stop_seen": false,
"tool_type": null
},
{
"completion_chars": 1151,
"end": "format_error: Could not parse 'Tool type' or 'Tool query' from the response. Ensure you output an <answer> block with 'Tool type:' and 'Tool query:'.",
"passed": false,
"stop_seen": false,
"tool_type": null
},
{
"completion_chars": 10895,
"end": "format_error: Could not parse 'Tool type' or 'Tool query' from the response. Ensure you output an <answer> block with 'Tool type:' and 'Tool query:'.",
"passed": false,
"stop_seen": false,
"tool_type": null
}
]
},
{
"difficulty": "hard",
"problem_id": "abc398_f",
"samples": [
{
"completion_chars": 2877,
"end": "wrong_answer",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"completion_chars": 2932,
"end": "format_error: Could not parse 'Tool type' or 'Tool query' from the response. Ensure you output an <answer> block with 'Tool type:' and 'Tool query:'.",
"passed": false,
"stop_seen": true,
"tool_type": null
},
{
"completion_chars": 715,
"end": "runtime_error",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"completion_chars": 2867,
"end": "wrong_answer",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"completion_chars": 2538,
"end": "wrong_answer",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"completion_chars": 2577,
"end": "wrong_answer",
"passed": false,
"stop_seen": true,
"tool_type": "final"
}
]
},
{
"difficulty": "easy",
"problem_id": "abc400_a",
"samples": [
{
"completion_chars": 328,
"end": "runtime_error",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"completion_chars": 917,
"end": "wrong_answer",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"completion_chars": 600,
"end": "wrong_answer",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"completion_chars": 742,
"end": "ok",
"passed": true,
"stop_seen": true,
"tool_type": "final"
},
{
"completion_chars": 808,
"end": "wrong_answer",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"completion_chars": 820,
"end": "ok",
"passed": true,
"stop_seen": true,
"tool_type": "final"
}
]
},
{
"difficulty": "hard",
"problem_id": "arc196_c",
"samples": [
{
"completion_chars": 5294,
"end": "runtime_error",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"completion_chars": 1689,
"end": "runtime_error",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"completion_chars": 5976,
"end": "format_error: final",
"passed": false,
"stop_seen": false,
"tool_type": "final"
},
{
"completion_chars": 4300,
"end": "wrong_answer",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"completion_chars": 4040,
"end": "wrong_answer",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"completion_chars": 5089,
"end": "wrong_answer",
"passed": false,
"stop_seen": true,
"tool_type": "final"
}
]
},
{
"difficulty": "medium",
"problem_id": "abc400_d",
"samples": [
{
"completion_chars": 8429,
"end": "wrong_answer",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"completion_chars": 7583,
"end": "runtime_error",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"completion_chars": 7201,
"end": "wrong_answer",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"completion_chars": 7008,
"end": "runtime_error",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"completion_chars": 2737,
"end": "format_error: final",
"passed": false,
"stop_seen": false,
"tool_type": "final"
},
{
"completion_chars": 7069,
"end": "runtime_error",
"passed": false,
"stop_seen": true,
"tool_type": "final"
}
]
},
{
"difficulty": "hard",
"problem_id": "abc398_g",
"samples": [
{
"completion_chars": 11630,
"end": "wrong_answer",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"completion_chars": 12637,
"end": "wrong_answer",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"completion_chars": 4093,
"end": "format_error: Could not parse 'Tool type' or 'Tool query' from the response. Ensure you output an <answer> block with 'Tool type:' and 'Tool query:'.",
"passed": false,
"stop_seen": false,
"tool_type": null
},
{
"completion_chars": 3088,
"end": "format_error: Could not parse 'Tool type' or 'Tool query' from the response. Ensure you output an <answer> block with 'Tool type:' and 'Tool query:'.",
"passed": false,
"stop_seen": false,
"tool_type": null
},
{
"completion_chars": 13122,
"end": "format_error: Could not parse 'Tool type' or 'Tool query' from the response. Ensure you output an <answer> block with 'Tool type:' and 'Tool query:'.",
"passed": false,
"stop_seen": false,
"tool_type": null
},
{
"completion_chars": 2073,
"end": "format_error: Could not parse 'Tool type' or 'Tool query' from the response. Ensure you output an <answer> block with 'Tool type:' and 'Tool query:'.",
"passed": false,
"stop_seen": false,
"tool_type": null
}
]
}
],
"eval_limit": 6,
"max_new_tokens": 1024,
"pass_at_1": 0.05555555555555555,
"passed_samples": 2,
"samples_per_problem": 6,
"tag": "stage1_group3",
"temperature": 1.0,
"tool_turn_limit": 1,
"top_p": 0.95,
"total_samples": 36
}

338
eval/full_schedule_endpoint_results.json Normal file
View File

@@ -0,0 +1,338 @@
{
"details": [
{
"difficulty": "hard",
"problem_id": "arc196_a",
"samples": [
{
"code_chars": 719,
"completion_chars": 828,
"end": "runtime_error",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"code_chars": 115,
"completion_chars": 2707,
"end": "wrong_answer",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"code_chars": 389,
"completion_chars": 1655,
"end": "timeout",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"code_chars": 214,
"completion_chars": 2386,
"end": "runtime_error",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"code_chars": 282,
"completion_chars": 2263,
"end": "runtime_error",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"code_chars": 303,
"completion_chars": 2072,
"end": "wrong_answer",
"passed": false,
"stop_seen": true,
"tool_type": "final"
}
]
},
{
"difficulty": "hard",
"problem_id": "abc398_f",
"samples": [
{
"code_chars": 165,
"completion_chars": 3251,
"end": "wrong_answer",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"code_chars": 160,
"completion_chars": 3358,
"end": "wrong_answer",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"code_chars": 877,
"completion_chars": 985,
"end": "runtime_error",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"code_chars": 499,
"completion_chars": 2528,
"end": "wrong_answer",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"code_chars": 396,
"completion_chars": 2252,
"end": "wrong_answer",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"code_chars": 327,
"completion_chars": 2724,
"end": "runtime_error",
"passed": false,
"stop_seen": true,
"tool_type": "final"
}
]
},
{
"difficulty": "easy",
"problem_id": "abc400_a",
"samples": [
{
"code_chars": 125,
"completion_chars": 12597,
"end": "wrong_answer",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"code_chars": 88,
"completion_chars": 12702,
"end": "wrong_answer",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"code_chars": 143,
"completion_chars": 12602,
"end": "wrong_answer",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"code_chars": 103,
"completion_chars": 12689,
"end": "wrong_answer",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"code_chars": 30,
"completion_chars": 12918,
"end": "wrong_answer",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"code_chars": 0,
"completion_chars": 2353,
"end": "format_error: Could not parse 'Tool type' or 'Tool query' from the response. Ensure you output an <answer> block with 'Tool type:' and 'Tool query:'.",
"passed": false,
"stop_seen": false,
"tool_type": null
}
]
},
{
"difficulty": "hard",
"problem_id": "arc196_c",
"samples": [
{
"code_chars": 111,
"completion_chars": 3537,
"end": "runtime_error",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"code_chars": 231,
"completion_chars": 2460,
"end": "wrong_answer",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"code_chars": 233,
"completion_chars": 3048,
"end": "wrong_answer",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"code_chars": 132,
"completion_chars": 3325,
"end": "wrong_answer",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"code_chars": 870,
"completion_chars": 979,
"end": "wrong_answer",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"code_chars": 389,
"completion_chars": 2591,
"end": "wrong_answer",
"passed": false,
"stop_seen": true,
"tool_type": "final"
}
]
},
{
"difficulty": "medium",
"problem_id": "abc400_d",
"samples": [
{
"code_chars": 733,
"completion_chars": 9798,
"end": "wrong_answer",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"code_chars": 0,
"completion_chars": 9689,
"end": "format_error: Could not parse 'Tool type' or 'Tool query' from the response. Ensure you output an <answer> block with 'Tool type:' and 'Tool query:'.",
"passed": false,
"stop_seen": false,
"tool_type": null
},
{
"code_chars": 1866,
"completion_chars": 7054,
"end": "runtime_error",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"code_chars": 1347,
"completion_chars": 8086,
"end": "runtime_error",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"code_chars": 0,
"completion_chars": 2217,
"end": "format_error: Could not parse 'Tool type' or 'Tool query' from the response. Ensure you output an <answer> block with 'Tool type:' and 'Tool query:'.",
"passed": false,
"stop_seen": false,
"tool_type": null
},
{
"code_chars": 1162,
"completion_chars": 8951,
"end": "runtime_error",
"passed": false,
"stop_seen": true,
"tool_type": "final"
}
]
},
{
"difficulty": "hard",
"problem_id": "abc398_g",
"samples": [
{
"code_chars": 0,
"completion_chars": 4543,
"end": "format_error: Could not parse 'Tool type' or 'Tool query' from the response. Ensure you output an <answer> block with 'Tool type:' and 'Tool query:'.",
"passed": false,
"stop_seen": false,
"tool_type": null
},
{
"code_chars": 2840,
"completion_chars": 3882,
"end": "timeout",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"code_chars": 347,
"completion_chars": 11776,
"end": "wrong_answer",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"code_chars": 0,
"completion_chars": 5113,
"end": "format_error: Could not parse 'Tool type' or 'Tool query' from the response. Ensure you output an <answer> block with 'Tool type:' and 'Tool query:'.",
"passed": false,
"stop_seen": false,
"tool_type": null
},
{
"code_chars": 791,
"completion_chars": 11156,
"end": "runtime_error",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"code_chars": 1219,
"completion_chars": 10183,
"end": "runtime_error",
"passed": false,
"stop_seen": true,
"tool_type": "final"
}
]
}
],
"eval_limit": 6,
"max_new_tokens": 1024,
"pass_at_1": 0.0,
"passed_samples": 0,
"samples_per_problem": 6,
"tag": "final",
"temperature": 1.0,
"tool_turn_limit": 1,
"top_p": 0.95,
"total_samples": 36
}

916
eval/selection_results.json Normal file
View File

@@ -0,0 +1,916 @@
{
"best": "stage1_group3",
"results": {
"after_sft": {
"details": [
{
"difficulty": "hard",
"problem_id": "arc196_a",
"samples": [
{
"completion_chars": 1731,
"end": "wrong_answer",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"completion_chars": 1841,
"end": "wrong_answer",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"completion_chars": 781,
"end": "runtime_error",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"completion_chars": 2276,
"end": "wrong_answer",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"completion_chars": 1543,
"end": "wrong_answer",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"completion_chars": 2677,
"end": "format_error: Could not parse 'Tool type' or 'Tool query' from the response. Ensure you output an <answer> block with 'Tool type:' and 'Tool query:'.",
"passed": false,
"stop_seen": false,
"tool_type": null
}
]
},
{
"difficulty": "hard",
"problem_id": "abc398_f",
"samples": [
{
"completion_chars": 3067,
"end": "format_error: Could not parse 'Tool type' or 'Tool query' from the response. Ensure you output an <answer> block with 'Tool type:' and 'Tool query:'.",
"passed": false,
"stop_seen": false,
"tool_type": null
},
{
"completion_chars": 2838,
"end": "wrong_answer",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"completion_chars": 599,
"end": "format_error: Could not parse 'Tool type' or 'Tool query' from the response. Ensure you output an <answer> block with 'Tool type:' and 'Tool query:'.",
"passed": false,
"stop_seen": false,
"tool_type": null
},
{
"completion_chars": 2426,
"end": "wrong_answer",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"completion_chars": 1308,
"end": "runtime_error",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"completion_chars": 1743,
"end": "wrong_answer",
"passed": false,
"stop_seen": true,
"tool_type": "final"
}
]
},
{
"difficulty": "easy",
"problem_id": "abc400_a",
"samples": [
{
"completion_chars": 376,
"end": "wrong_answer",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"completion_chars": 520,
"end": "wrong_answer",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"completion_chars": 725,
"end": "ok",
"passed": true,
"stop_seen": true,
"tool_type": "final"
},
{
"completion_chars": 666,
"end": "wrong_answer",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"completion_chars": 294,
"end": "format_error: Could not parse 'Tool type' or 'Tool query' from the response. Ensure you output an <answer> block with 'Tool type:' and 'Tool query:'.",
"passed": false,
"stop_seen": false,
"tool_type": null
},
{
"completion_chars": 431,
"end": "wrong_answer",
"passed": false,
"stop_seen": true,
"tool_type": "final"
}
]
},
{
"difficulty": "hard",
"problem_id": "arc196_c",
"samples": [
{
"completion_chars": 10417,
"end": "runtime_error",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"completion_chars": 11700,
"end": "runtime_error",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"completion_chars": 1704,
"end": "format_error: Could not parse 'Tool type' or 'Tool query' from the response. Ensure you output an <answer> block with 'Tool type:' and 'Tool query:'.",
"passed": false,
"stop_seen": false,
"tool_type": null
},
{
"completion_chars": 3592,
"end": "format_error: Could not parse 'Tool type' or 'Tool query' from the response. Ensure you output an <answer> block with 'Tool type:' and 'Tool query:'.",
"passed": false,
"stop_seen": false,
"tool_type": null
},
{
"completion_chars": 12478,
"end": "runtime_error",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"completion_chars": 11832,
"end": "wrong_answer",
"passed": false,
"stop_seen": true,
"tool_type": "final"
}
]
},
{
"difficulty": "medium",
"problem_id": "abc400_d",
"samples": [
{
"completion_chars": 5287,
"end": "runtime_error",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"completion_chars": 10882,
"end": "runtime_error",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"completion_chars": 1065,
"end": "format_error: Could not parse 'Tool type' or 'Tool query' from the response. Ensure you output an <answer> block with 'Tool type:' and 'Tool query:'.",
"passed": false,
"stop_seen": false,
"tool_type": null
},
{
"completion_chars": 1647,
"end": "format_error: Could not parse 'Tool type' or 'Tool query' from the response. Ensure you output an <answer> block with 'Tool type:' and 'Tool query:'.",
"passed": false,
"stop_seen": false,
"tool_type": null
},
{
"completion_chars": 13229,
"end": "format_error: Could not parse 'Tool type' or 'Tool query' from the response. Ensure you output an <answer> block with 'Tool type:' and 'Tool query:'.",
"passed": false,
"stop_seen": false,
"tool_type": null
},
{
"completion_chars": 9263,
"end": "timeout",
"passed": false,
"stop_seen": true,
"tool_type": "final"
}
]
},
{
"difficulty": "hard",
"problem_id": "abc398_g",
"samples": [
{
"completion_chars": 12369,
"end": "wrong_answer",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"completion_chars": 10642,
"end": "runtime_error",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"completion_chars": 11360,
"end": "wrong_answer",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"completion_chars": 2075,
"end": "format_error: Could not parse 'Tool type' or 'Tool query' from the response. Ensure you output an <answer> block with 'Tool type:' and 'Tool query:'.",
"passed": false,
"stop_seen": false,
"tool_type": null
},
{
"completion_chars": 11910,
"end": "wrong_answer",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"completion_chars": 604,
"end": "format_error: Could not parse 'Tool type' or 'Tool query' from the response. Ensure you output an <answer> block with 'Tool type:' and 'Tool query:'.",
"passed": false,
"stop_seen": false,
"tool_type": null
}
]
}
],
"eval_limit": 6,
"max_new_tokens": 1024,
"pass_at_1": 0.027777777777777776,
"passed_samples": 1,
"samples_per_problem": 6,
"tag": "after_sft",
"temperature": 1.0,
"tool_turn_limit": 1,
"top_p": 0.95,
"total_samples": 36
},
"stage1_group3": {
"details": [
{
"difficulty": "hard",
"problem_id": "arc196_a",
"samples": [
{
"completion_chars": 11690,
"end": "wrong_answer",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"completion_chars": 12444,
"end": "wrong_answer",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"completion_chars": 12570,
"end": "wrong_answer",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"completion_chars": 12044,
"end": "format_error: Could not parse 'Tool type' or 'Tool query' from the response. Ensure you output an <answer> block with 'Tool type:' and 'Tool query:'.",
"passed": false,
"stop_seen": false,
"tool_type": null
},
{
"completion_chars": 1151,
"end": "format_error: Could not parse 'Tool type' or 'Tool query' from the response. Ensure you output an <answer> block with 'Tool type:' and 'Tool query:'.",
"passed": false,
"stop_seen": false,
"tool_type": null
},
{
"completion_chars": 10895,
"end": "format_error: Could not parse 'Tool type' or 'Tool query' from the response. Ensure you output an <answer> block with 'Tool type:' and 'Tool query:'.",
"passed": false,
"stop_seen": false,
"tool_type": null
}
]
},
{
"difficulty": "hard",
"problem_id": "abc398_f",
"samples": [
{
"completion_chars": 2877,
"end": "wrong_answer",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"completion_chars": 2932,
"end": "format_error: Could not parse 'Tool type' or 'Tool query' from the response. Ensure you output an <answer> block with 'Tool type:' and 'Tool query:'.",
"passed": false,
"stop_seen": true,
"tool_type": null
},
{
"completion_chars": 715,
"end": "runtime_error",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"completion_chars": 2867,
"end": "wrong_answer",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"completion_chars": 2538,
"end": "wrong_answer",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"completion_chars": 2577,
"end": "wrong_answer",
"passed": false,
"stop_seen": true,
"tool_type": "final"
}
]
},
{
"difficulty": "easy",
"problem_id": "abc400_a",
"samples": [
{
"completion_chars": 328,
"end": "runtime_error",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"completion_chars": 917,
"end": "wrong_answer",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"completion_chars": 600,
"end": "wrong_answer",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"completion_chars": 742,
"end": "ok",
"passed": true,
"stop_seen": true,
"tool_type": "final"
},
{
"completion_chars": 808,
"end": "wrong_answer",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"completion_chars": 820,
"end": "ok",
"passed": true,
"stop_seen": true,
"tool_type": "final"
}
]
},
{
"difficulty": "hard",
"problem_id": "arc196_c",
"samples": [
{
"completion_chars": 5294,
"end": "runtime_error",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"completion_chars": 1689,
"end": "runtime_error",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"completion_chars": 5976,
"end": "format_error: final",
"passed": false,
"stop_seen": false,
"tool_type": "final"
},
{
"completion_chars": 4300,
"end": "wrong_answer",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"completion_chars": 4040,
"end": "wrong_answer",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"completion_chars": 5089,
"end": "wrong_answer",
"passed": false,
"stop_seen": true,
"tool_type": "final"
}
]
},
{
"difficulty": "medium",
"problem_id": "abc400_d",
"samples": [
{
"completion_chars": 8429,
"end": "wrong_answer",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"completion_chars": 7583,
"end": "runtime_error",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"completion_chars": 7201,
"end": "wrong_answer",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"completion_chars": 7008,
"end": "runtime_error",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"completion_chars": 2737,
"end": "format_error: final",
"passed": false,
"stop_seen": false,
"tool_type": "final"
},
{
"completion_chars": 7069,
"end": "runtime_error",
"passed": false,
"stop_seen": true,
"tool_type": "final"
}
]
},
{
"difficulty": "hard",
"problem_id": "abc398_g",
"samples": [
{
"completion_chars": 11630,
"end": "wrong_answer",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"completion_chars": 12637,
"end": "wrong_answer",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"completion_chars": 4093,
"end": "format_error: Could not parse 'Tool type' or 'Tool query' from the response. Ensure you output an <answer> block with 'Tool type:' and 'Tool query:'.",
"passed": false,
"stop_seen": false,
"tool_type": null
},
{
"completion_chars": 3088,
"end": "format_error: Could not parse 'Tool type' or 'Tool query' from the response. Ensure you output an <answer> block with 'Tool type:' and 'Tool query:'.",
"passed": false,
"stop_seen": false,
"tool_type": null
},
{
"completion_chars": 13122,
"end": "format_error: Could not parse 'Tool type' or 'Tool query' from the response. Ensure you output an <answer> block with 'Tool type:' and 'Tool query:'.",
"passed": false,
"stop_seen": false,
"tool_type": null
},
{
"completion_chars": 2073,
"end": "format_error: Could not parse 'Tool type' or 'Tool query' from the response. Ensure you output an <answer> block with 'Tool type:' and 'Tool query:'.",
"passed": false,
"stop_seen": false,
"tool_type": null
}
]
}
],
"eval_limit": 6,
"max_new_tokens": 1024,
"pass_at_1": 0.05555555555555555,
"passed_samples": 2,
"samples_per_problem": 6,
"tag": "stage1_group3",
"temperature": 1.0,
"tool_turn_limit": 1,
"top_p": 0.95,
"total_samples": 36
},
"stage2_group1": {
"details": [
{
"difficulty": "hard",
"problem_id": "arc196_a",
"samples": [
{
"completion_chars": 11873,
"end": "wrong_answer",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"completion_chars": 4103,
"end": "format_error: Could not parse 'Tool type' or 'Tool query' from the response. Ensure you output an <answer> block with 'Tool type:' and 'Tool query:'.",
"passed": false,
"stop_seen": false,
"tool_type": null
},
{
"completion_chars": 10118,
"end": "format_error: Could not parse 'Tool type' or 'Tool query' from the response. Ensure you output an <answer> block with 'Tool type:' and 'Tool query:'.",
"passed": false,
"stop_seen": false,
"tool_type": null
},
{
"completion_chars": 11800,
"end": "wrong_answer",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"completion_chars": 2309,
"end": "format_error: Could not parse 'Tool type' or 'Tool query' from the response. Ensure you output an <answer> block with 'Tool type:' and 'Tool query:'.",
"passed": false,
"stop_seen": false,
"tool_type": null
},
{
"completion_chars": 2870,
"end": "format_error: final",
"passed": false,
"stop_seen": false,
"tool_type": "final"
}
]
},
{
"difficulty": "hard",
"problem_id": "abc398_f",
"samples": [
{
"completion_chars": 1513,
"end": "wrong_answer",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"completion_chars": 1075,
"end": "wrong_answer",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"completion_chars": 1550,
"end": "wrong_answer",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"completion_chars": 1186,
"end": "wrong_answer",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"completion_chars": 1775,
"end": "format_error: Could not parse 'Tool type' or 'Tool query' from the response. Ensure you output an <answer> block with 'Tool type:' and 'Tool query:'.",
"passed": false,
"stop_seen": false,
"tool_type": null
},
{
"completion_chars": 399,
"end": "wrong_answer",
"passed": false,
"stop_seen": true,
"tool_type": "final"
}
]
},
{
"difficulty": "easy",
"problem_id": "abc400_a",
"samples": [
{
"completion_chars": 478,
"end": "format_error: Could not parse 'Tool type' or 'Tool query' from the response. Ensure you output an <answer> block with 'Tool type:' and 'Tool query:'.",
"passed": false,
"stop_seen": false,
"tool_type": null
},
{
"completion_chars": 466,
"end": "wrong_answer",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"completion_chars": 949,
"end": "format_error: Could not parse 'Tool type' or 'Tool query' from the response. Ensure you output an <answer> block with 'Tool type:' and 'Tool query:'.",
"passed": false,
"stop_seen": false,
"tool_type": null
},
{
"completion_chars": 556,
"end": "ok",
"passed": true,
"stop_seen": true,
"tool_type": "final"
},
{
"completion_chars": 328,
"end": "wrong_answer",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"completion_chars": 434,
"end": "wrong_answer",
"passed": false,
"stop_seen": true,
"tool_type": "final"
}
]
},
{
"difficulty": "hard",
"problem_id": "arc196_c",
"samples": [
{
"completion_chars": 5189,
"end": "wrong_answer",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"completion_chars": 4221,
"end": "wrong_answer",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"completion_chars": 3838,
"end": "wrong_answer",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"completion_chars": 1365,
"end": "format_error: final",
"passed": false,
"stop_seen": false,
"tool_type": "final"
},
{
"completion_chars": 4021,
"end": "wrong_answer",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"completion_chars": 4585,
"end": "wrong_answer",
"passed": false,
"stop_seen": true,
"tool_type": "final"
}
]
},
{
"difficulty": "medium",
"problem_id": "abc400_d",
"samples": [
{
"completion_chars": 1449,
"end": "format_error: Could not parse 'Tool type' or 'Tool query' from the response. Ensure you output an <answer> block with 'Tool type:' and 'Tool query:'.",
"passed": false,
"stop_seen": false,
"tool_type": null
},
{
"completion_chars": 13228,
"end": "format_error: Could not parse 'Tool type' or 'Tool query' from the response. Ensure you output an <answer> block with 'Tool type:' and 'Tool query:'.",
"passed": false,
"stop_seen": false,
"tool_type": null
},
{
"completion_chars": 7164,
"end": "runtime_error",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"completion_chars": 10712,
"end": "runtime_error",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"completion_chars": 9464,
"end": "wrong_answer",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"completion_chars": 5995,
"end": "runtime_error",
"passed": false,
"stop_seen": true,
"tool_type": "final"
}
]
},
{
"difficulty": "hard",
"problem_id": "abc398_g",
"samples": [
{
"completion_chars": 10181,
"end": "runtime_error",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"completion_chars": 11037,
"end": "wrong_answer",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"completion_chars": 2580,
"end": "format_error: Could not parse 'Tool type' or 'Tool query' from the response. Ensure you output an <answer> block with 'Tool type:' and 'Tool query:'.",
"passed": false,
"stop_seen": false,
"tool_type": null
},
{
"completion_chars": 10968,
"end": "wrong_answer",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"completion_chars": 12204,
"end": "wrong_answer",
"passed": false,
"stop_seen": true,
"tool_type": "final"
},
{
"completion_chars": 2073,
"end": "format_error: Could not parse 'Tool type' or 'Tool query' from the response. Ensure you output an <answer> block with 'Tool type:' and 'Tool query:'.",
"passed": false,
"stop_seen": false,
"tool_type": null
}
]
}
],
"eval_limit": 6,
"max_new_tokens": 1024,
"pass_at_1": 0.027777777777777776,
"passed_samples": 1,
"samples_per_problem": 6,
"tag": "stage2_group1",
"temperature": 1.0,
"tool_turn_limit": 1,
"top_p": 0.95,
"total_samples": 36
}
},
"summary": {
"after_sft": 0.027777777777777776,
"stage1_group3": 0.05555555555555555,
"stage2_group1": 0.027777777777777776
}
}

770
eval/sft_log.json Normal file
View File

@@ -0,0 +1,770 @@
[
{
"grad_norm": 1.040154218673706,
"index": 1,
"loss": 0.48266512155532837,
"nll": 0.48266512155532837,
"problem_id": "438_A. The Child and Toy",
"skipped": null
},
{
"grad_norm": 1.1882520914077759,
"index": 2,
"loss": 0.5828834176063538,
"nll": 0.5828834176063538,
"problem_id": "300_A. Array",
"skipped": null
},
{
"grad_norm": 1.9626847505569458,
"index": 3,
"loss": 1.1390048265457153,
"nll": 1.1390048265457153,
"problem_id": "478_A. Initial Bet",
"skipped": null
},
{
"grad_norm": 1.5171997547149658,
"index": 4,
"loss": 0.9680904150009155,
"nll": 0.9680904150009155,
"problem_id": "676_A. Nicholas and Permutation",
"skipped": null
},
{
"grad_norm": 0.3860478699207306,
"index": 5,
"loss": 0.39563682675361633,
"nll": 0.39563682675361633,
"problem_id": "1031_B. Curiosity Has No Limits",
"skipped": null
},
{
"grad_norm": 1.020121455192566,
"index": 6,
"loss": 0.8472175002098083,
"nll": 0.8472175002098083,
"problem_id": "940_A. Points on the line",
"skipped": null
},
{
"grad_norm": 0.4967013895511627,
"index": 7,
"loss": 0.5509471893310547,
"nll": 0.5509471893310547,
"problem_id": "1080_C. Masha and two friends",
"skipped": null
},
{
"grad_norm": 1.479300856590271,
"index": 8,
"loss": 0.8386653661727905,
"nll": 0.8386653661727905,
"problem_id": "38_C. Blinds",
"skipped": null
},
{
"grad_norm": 0.8893219828605652,
"index": 9,
"loss": 0.3628019392490387,
"nll": 0.3628019392490387,
"problem_id": "489_B. BerSU Ball",
"skipped": null
},
{
"grad_norm": 1.4400527477264404,
"index": 10,
"loss": 0.9306145310401917,
"nll": 0.9306145310401917,
"problem_id": "787_A. The Monster",
"skipped": null
},
{
"grad_norm": 1.0569334030151367,
"index": 11,
"loss": 0.6781665086746216,
"nll": 0.6781665086746216,
"problem_id": "1070_D. Garbage Disposal",
"skipped": null
},
{
"grad_norm": 1.9829460382461548,
"index": 12,
"loss": 1.084402322769165,
"nll": 1.084402322769165,
"problem_id": "805_A. Fake NP",
"skipped": null
},
{
"grad_norm": 0.910430908203125,
"index": 13,
"loss": 0.7472644448280334,
"nll": 0.7472644448280334,
"problem_id": "1305_B. Kuroni and Simple Strings",
"skipped": null
},
{
"grad_norm": 0.8546664714813232,
"index": 14,
"loss": 0.4409652650356293,
"nll": 0.4409652650356293,
"problem_id": "742_A. Arpa\u2019s hard exam and Mehrdad\u2019s naive cheat",
"skipped": null
},
{
"grad_norm": 1.0713262557983398,
"index": 15,
"loss": 0.9237834215164185,
"nll": 0.9237834215164185,
"problem_id": "358_B. Dima and Text Messages",
"skipped": null
},
{
"grad_norm": 1.3963273763656616,
"index": 16,
"loss": 0.7768023610115051,
"nll": 0.7768023610115051,
"problem_id": "1443_A. Kids Seating",
"skipped": null
},
{
"grad_norm": 1.0952928066253662,
"index": 17,
"loss": 0.8175995945930481,
"nll": 0.8175995945930481,
"problem_id": "604_B. More Cowbell",
"skipped": null
},
{
"grad_norm": 1.6387556791305542,
"index": 18,
"loss": 1.1116256713867188,
"nll": 1.1116256713867188,
"problem_id": "1043_A. Elections",
"skipped": null
},
{
"grad_norm": 0.9755963683128357,
"index": 19,
"loss": 0.7564535737037659,
"nll": 0.7564535737037659,
"problem_id": "1054_B. Appending Mex",
"skipped": null
},
{
"grad_norm": 0.8682615756988525,
"index": 20,
"loss": 0.785297691822052,
"nll": 0.785297691822052,
"problem_id": "248_B. Chilly Willy",
"skipped": null
},
{
"grad_norm": 0.7661585807800293,
"index": 21,
"loss": 0.6494926810264587,
"nll": 0.6494926810264587,
"problem_id": "985_A. Chess Placing",
"skipped": null
},
{
"grad_norm": 2.292353630065918,
"index": 22,
"loss": 0.8781017065048218,
"nll": 0.8781017065048218,
"problem_id": "110_B. Lucky String",
"skipped": null
},
{
"grad_norm": 0.833644449710846,
"index": 23,
"loss": 0.504396378993988,
"nll": 0.504396378993988,
"problem_id": "519_B. A and B and Compilation Errors",
"skipped": null
},
{
"grad_norm": 0.9993244409561157,
"index": 24,
"loss": 0.8234552145004272,
"nll": 0.8234552145004272,
"problem_id": "615_A. Bulbs",
"skipped": null
},
{
"grad_norm": 1.3748852014541626,
"index": 25,
"loss": 0.5862762928009033,
"nll": 0.5862762928009033,
"problem_id": "270_B. Multithreading",
"skipped": null
},
{
"grad_norm": 1.3495038747787476,
"index": 26,
"loss": 1.5716842412948608,
"nll": 1.5716842412948608,
"problem_id": "272_C. Dima and Staircase",
"skipped": null
},
{
"grad_norm": 0.9883641600608826,
"index": 27,
"loss": 0.5327346324920654,
"nll": 0.5327346324920654,
"problem_id": "5_A. Chat Server's Outgoing Traffic",
"skipped": null
},
{
"grad_norm": 0.7346091270446777,
"index": 28,
"loss": 0.721221923828125,
"nll": 0.721221923828125,
"problem_id": "1408_B. Arrays Sum",
"skipped": null
},
{
"grad_norm": 1.3178147077560425,
"index": 29,
"loss": 0.7296963334083557,
"nll": 0.7296963334083557,
"problem_id": "177_D1. Encrypting Messages",
"skipped": null
},
{
"grad_norm": 1.020062804222107,
"index": 30,
"loss": 0.5247489213943481,
"nll": 0.5247489213943481,
"problem_id": "282_B. Painting Eggs",
"skipped": null
},
{
"grad_norm": 0.8763545155525208,
"index": 31,
"loss": 0.5632577538490295,
"nll": 0.5632577538490295,
"problem_id": "1334_A. Level Statistics",
"skipped": null
},
{
"grad_norm": 0.9953303337097168,
"index": 32,
"loss": 0.3932126760482788,
"nll": 0.3932126760482788,
"problem_id": "630_A. Again Twenty Five!",
"skipped": null
},
{
"grad_norm": 1.3474981784820557,
"index": 33,
"loss": 0.514382004737854,
"nll": 0.514382004737854,
"problem_id": "1366_A. Shovels and Swords",
"skipped": null
},
{
"grad_norm": 0.7742969393730164,
"index": 34,
"loss": 0.5816912055015564,
"nll": 0.5816912055015564,
"problem_id": "448_B. Suffix Structures",
"skipped": null
},
{
"grad_norm": 1.562877893447876,
"index": 35,
"loss": 0.8283543586730957,
"nll": 0.8283543586730957,
"problem_id": "1353_C. Board Moves",
"skipped": null
},
{
"grad_norm": 0.783785343170166,
"index": 36,
"loss": 0.3817444145679474,
"nll": 0.3817444145679474,
"problem_id": "259_B. Little Elephant and Magic Square",
"skipped": null
},
{
"grad_norm": 0.7053418755531311,
"index": 37,
"loss": 0.4648408889770508,
"nll": 0.4648408889770508,
"problem_id": "687_A. NP-Hard Problem",
"skipped": null
},
{
"grad_norm": 0.8954440355300903,
"index": 38,
"loss": 0.5581481456756592,
"nll": 0.5581481456756592,
"problem_id": "1121_A. Technogoblet of Fire",
"skipped": null
},
{
"grad_norm": 1.3505918979644775,
"index": 39,
"loss": 0.5477667450904846,
"nll": 0.5477667450904846,
"problem_id": "918_A. Eleven",
"skipped": null
},
{
"grad_norm": 1.0709880590438843,
"index": 40,
"loss": 0.7576397657394409,
"nll": 0.7576397657394409,
"problem_id": "1084_C. The Fair Nut and String",
"skipped": null
},
{
"grad_norm": 1.2410813570022583,
"index": 41,
"loss": 0.981418788433075,
"nll": 0.981418788433075,
"problem_id": "545_C. Woodcutters",
"skipped": null
},
{
"grad_norm": 0.44853660464286804,
"index": 42,
"loss": 0.20538830757141113,
"nll": 0.20538830757141113,
"problem_id": "1152_A. Neko Finds Grapes",
"skipped": null
},
{
"grad_norm": 0.7902032732963562,
"index": 43,
"loss": 0.43099430203437805,
"nll": 0.43099430203437805,
"problem_id": "385_A. Bear and Raspberry",
"skipped": null
},
{
"grad_norm": 0.8370153307914734,
"index": 44,
"loss": 0.5004207491874695,
"nll": 0.5004207491874695,
"problem_id": "1392_B. Omkar and Infinity Clock",
"skipped": null
},
{
"grad_norm": 0.6053890585899353,
"index": 45,
"loss": 0.527194082736969,
"nll": 0.527194082736969,
"problem_id": "1301_B. Motarack's Birthday",
"skipped": null
},
{
"grad_norm": 1.0937732458114624,
"index": 46,
"loss": 0.703790545463562,
"nll": 0.703790545463562,
"problem_id": "494_A. Treasure",
"skipped": null
},
{
"grad_norm": 0.902428150177002,
"index": 47,
"loss": 0.42323175072669983,
"nll": 0.42323175072669983,
"problem_id": "1366_B. Shuffle",
"skipped": null
},
{
"grad_norm": 0.6949153542518616,
"index": 48,
"loss": 0.37455669045448303,
"nll": 0.37455669045448303,
"problem_id": "69_A. Young Physicist",
"skipped": null
},
{
"grad_norm": 1.3904484510421753,
"index": 49,
"loss": 0.7064117193222046,
"nll": 0.7064117193222046,
"problem_id": "125_A. Measuring Lengths in Baden",
"skipped": null
},
{
"grad_norm": 1.2604994773864746,
"index": 50,
"loss": 0.6908637285232544,
"nll": 0.6908637285232544,
"problem_id": "45_I. TCMCF+++",
"skipped": null
},
{
"grad_norm": 1.3847407102584839,
"index": 51,
"loss": 0.6546027660369873,
"nll": 0.6546027660369873,
"problem_id": "740_B. Alyona and flowers",
"skipped": null
},
{
"grad_norm": 1.031480073928833,
"index": 52,
"loss": 0.6020389795303345,
"nll": 0.6020389795303345,
"problem_id": "136_B. Ternary Logic",
"skipped": null
},
{
"grad_norm": 0.9329264760017395,
"index": 53,
"loss": 0.766955554485321,
"nll": 0.766955554485321,
"problem_id": "1139_C. Edgy Trees",
"skipped": null
},
{
"grad_norm": 0.34977203607559204,
"index": 54,
"loss": 0.44151249527931213,
"nll": 0.44151249527931213,
"problem_id": "1481_B. New Colony",
"skipped": null
},
{
"grad_norm": 0.8771538734436035,
"index": 55,
"loss": 0.7079184055328369,
"nll": 0.7079184055328369,
"problem_id": "1382_A. Common Subsequence",
"skipped": null
},
{
"grad_norm": 1.0364067554473877,
"index": 56,
"loss": 0.8011530637741089,
"nll": 0.8011530637741089,
"problem_id": "248_A. Cupboards",
"skipped": null
},
{
"grad_norm": 0.8864939212799072,
"index": 57,
"loss": 0.5355629324913025,
"nll": 0.5355629324913025,
"problem_id": "268_A. Games",
"skipped": null
},
{
"grad_norm": 0.47688010334968567,
"index": 58,
"loss": 0.509407639503479,
"nll": 0.509407639503479,
"problem_id": "618_B. Guess the Permutation",
"skipped": null
},
{
"grad_norm": 1.1146557331085205,
"index": 59,
"loss": 0.9011950492858887,
"nll": 0.9011950492858887,
"problem_id": "1181_A. Chunga-Changa",
"skipped": null
},
{
"grad_norm": 0.5178598761558533,
"index": 60,
"loss": 0.2951464056968689,
"nll": 0.2951464056968689,
"problem_id": "1421_B. Putting Bricks in the Wall",
"skipped": null
},
{
"grad_norm": 0.9971789121627808,
"index": 61,
"loss": 0.5864426493644714,
"nll": 0.5864426493644714,
"problem_id": "554_C. Kyoya and Colored Balls",
"skipped": null
},
{
"grad_norm": 0.699429452419281,
"index": 62,
"loss": 0.4205191731452942,
"nll": 0.4205191731452942,
"problem_id": "1203_A. Circle of Students",
"skipped": null
},
{
"grad_norm": 0.7046372294425964,
"index": 63,
"loss": 0.8773186206817627,
"nll": 0.8773186206817627,
"problem_id": "1427_B. Chess Cheater",
"skipped": null
},
{
"grad_norm": 1.2316917181015015,
"index": 64,
"loss": 0.7916744351387024,
"nll": 0.7916744351387024,
"problem_id": "670_B. Game of Robots",
"skipped": null
},
{
"grad_norm": 0.7533033490180969,
"index": 65,
"loss": 0.5084953308105469,
"nll": 0.5084953308105469,
"problem_id": "1000_A. Codehorses T-shirts",
"skipped": null
},
{
"grad_norm": 0.4042784869670868,
"index": 66,
"loss": 0.3695657253265381,
"nll": 0.3695657253265381,
"problem_id": "1264_A. Beautiful Regional Contest",
"skipped": null
},
{
"grad_norm": 1.0112096071243286,
"index": 67,
"loss": 0.8849809169769287,
"nll": 0.8849809169769287,
"problem_id": "538_B. Quasi Binary",
"skipped": null
},
{
"grad_norm": 0.7484896183013916,
"index": 68,
"loss": 0.5725398659706116,
"nll": 0.5725398659706116,
"problem_id": "868_A. Bark to Unlock",
"skipped": null
},
{
"grad_norm": 0.4507679045200348,
"index": 69,
"loss": 0.40581223368644714,
"nll": 0.40581223368644714,
"problem_id": "99_B. Help Chef Gerasim",
"skipped": null
},
{
"grad_norm": 0.6510177850723267,
"index": 70,
"loss": 0.5508701205253601,
"nll": 0.5508701205253601,
"problem_id": "931_A. Friends Meeting",
"skipped": null
},
{
"grad_norm": 0.9200932383537292,
"index": 71,
"loss": 0.6416708827018738,
"nll": 0.6416708827018738,
"problem_id": "870_A. Search for Pretty Integers",
"skipped": null
},
{
"grad_norm": 0.8513489365577698,
"index": 72,
"loss": 1.0005862712860107,
"nll": 1.0005862712860107,
"problem_id": "1191_C. Tokitsukaze and Discard Items",
"skipped": null
},
{
"grad_norm": 0.4327949285507202,
"index": 73,
"loss": 0.5023372173309326,
"nll": 0.5023372173309326,
"problem_id": "505_B. Mr. Kitayuta's Colorful Graph",
"skipped": null
},
{
"grad_norm": 0.6228141784667969,
"index": 74,
"loss": 0.482375830411911,
"nll": 0.482375830411911,
"problem_id": "365_A. Good Number",
"skipped": null
},
{
"grad_norm": 0.9539189338684082,
"index": 75,
"loss": 0.5643600225448608,
"nll": 0.5643600225448608,
"problem_id": "513_A. Game",
"skipped": null
},
{
"grad_norm": 0.8559263944625854,
"index": 76,
"loss": 1.1494770050048828,
"nll": 1.1494770050048828,
"problem_id": "1159_A. A pile of stones",
"skipped": null
},
{
"grad_norm": 0.9862070083618164,
"index": 77,
"loss": 1.1389682292938232,
"nll": 1.1389682292938232,
"problem_id": "1398_A. Bad Triangle",
"skipped": null
},
{
"grad_norm": 1.9144845008850098,
"index": 78,
"loss": 0.7307466268539429,
"nll": 0.7307466268539429,
"problem_id": "483_C. Diverse Permutation",
"skipped": null
},
{
"grad_norm": 1.0251142978668213,
"index": 79,
"loss": 0.7849968075752258,
"nll": 0.7849968075752258,
"problem_id": "673_A. Bear and Game",
"skipped": null
},
{
"grad_norm": 1.3851652145385742,
"index": 80,
"loss": 0.9004217386245728,
"nll": 0.9004217386245728,
"problem_id": "604_A. Uncowed Forces",
"skipped": null
},
{
"grad_norm": 0.8206305503845215,
"index": 81,
"loss": 0.513279914855957,
"nll": 0.513279914855957,
"problem_id": "1059_A. Cashier",
"skipped": null
},
{
"grad_norm": 0.4920770823955536,
"index": 82,
"loss": 0.2822416424751282,
"nll": 0.2822416424751282,
"problem_id": "835_A. Key races",
"skipped": null
},
{
"grad_norm": 0.789677083492279,
"index": 83,
"loss": 0.6259894967079163,
"nll": 0.6259894967079163,
"problem_id": "1490_D. Permutation Transformation",
"skipped": null
},
{
"grad_norm": 0.9807916879653931,
"index": 84,
"loss": 0.98895663022995,
"nll": 0.98895663022995,
"problem_id": "443_B. Kolya and Tandem Repeat",
"skipped": null
},
{
"grad_norm": 0.6448408365249634,
"index": 85,
"loss": 0.4732319116592407,
"nll": 0.4732319116592407,
"problem_id": "580_B. Kefa and Company",
"skipped": null
},
{
"grad_norm": 0.6379855871200562,
"index": 86,
"loss": 0.470840722322464,
"nll": 0.470840722322464,
"problem_id": "143_A. Help Vasilisa the Wise 2",
"skipped": null
},
{
"grad_norm": 0.816592276096344,
"index": 87,
"loss": 0.8404387831687927,
"nll": 0.8404387831687927,
"problem_id": "1011_A. Stages",
"skipped": null
},
{
"grad_norm": 0.6529431343078613,
"index": 88,
"loss": 0.6879411935806274,
"nll": 0.6879411935806274,
"problem_id": "1269_B. Modulo Equality",
"skipped": null
},
{
"grad_norm": 0.6517459154129028,
"index": 89,
"loss": 0.5100618004798889,
"nll": 0.5100618004798889,
"problem_id": "1543_B. Customising the Track",
"skipped": null
},
{
"grad_norm": 0.5063552260398865,
"index": 90,
"loss": 0.49379613995552063,
"nll": 0.49379613995552063,
"problem_id": "177_C1. Party",
"skipped": null
},
{
"grad_norm": 1.0049214363098145,
"index": 91,
"loss": 0.4494905471801758,
"nll": 0.4494905471801758,
"problem_id": "379_A. New Year Candles",
"skipped": null
},
{
"grad_norm": 0.5677995681762695,
"index": 92,
"loss": 0.5990645885467529,
"nll": 0.5990645885467529,
"problem_id": "1474_B. Different Divisors",
"skipped": null
},
{
"grad_norm": 0.9032794237136841,
"index": 93,
"loss": 0.5278613567352295,
"nll": 0.5278613567352295,
"problem_id": "155_A. I_love_%username%",
"skipped": null
},
{
"grad_norm": 0.44249868392944336,
"index": 94,
"loss": 0.372690349817276,
"nll": 0.372690349817276,
"problem_id": "673_C. Bear and Colors",
"skipped": null
},
{
"grad_norm": 0.7681690454483032,
"index": 95,
"loss": 0.7501130104064941,
"nll": 0.7501130104064941,
"problem_id": "381_A. Sereja and Dima",
"skipped": null
},
{
"grad_norm": 1.172059416770935,
"index": 96,
"loss": 0.8275896310806274,
"nll": 0.8275896310806274,
"problem_id": "39_H. Multiplication Table",
"skipped": null
}
]

2010
eval/training_stage_log.json Normal file
View File

File diff suppressed because it is too large Load Diff