{ "count": 100, "solved": 79, "solved_rate": 0.79, "total_model_candidates": 25600, "mean_candidate_budget_per_problem": 256, "max_candidate_budget_per_problem": 256, "min_candidate_budget_per_problem": 256, "mean_candidates_until_solution": 37.51898734177215, "median_candidates_until_solution": 21, "max_candidates_until_solution": 164, "source_budgets": { "outputs/eval/sample_shaped/shard_0.jsonl": { "rows": 100, "candidates": 1600 }, "outputs/eval/sample_shaped/shard_1.jsonl": { "rows": 100, "candidates": 1600 }, "outputs/eval/sample_shaped/shard_2.jsonl": { "rows": 100, "candidates": 1600 }, "outputs/eval/sample_shaped/shard_3.jsonl": { "rows": 100, "candidates": 1600 }, "outputs/eval/sample_shaped/shard_4.jsonl": { "rows": 100, "candidates": 1600 }, "outputs/eval/sample_shaped/shard_5.jsonl": { "rows": 100, "candidates": 1600 }, "outputs/eval/sample_shaped/shard_6.jsonl": { "rows": 100, "candidates": 1600 }, "outputs/eval/sample_shaped/shard_7.jsonl": { "rows": 100, "candidates": 1600 }, "outputs/eval/sample_ch/shard_0.jsonl": { "rows": 100, "candidates": 1600 }, "outputs/eval/sample_ch/shard_1.jsonl": { "rows": 100, "candidates": 1600 }, "outputs/eval/sample_ch/shard_2.jsonl": { "rows": 100, "candidates": 1600 }, "outputs/eval/sample_ch/shard_3.jsonl": { "rows": 100, "candidates": 1600 }, "outputs/eval/sample_ch/shard_4.jsonl": { "rows": 100, "candidates": 1600 }, "outputs/eval/sample_ch/shard_5.jsonl": { "rows": 100, "candidates": 1600 }, "outputs/eval/sample_ch/shard_6.jsonl": { "rows": 100, "candidates": 1600 }, "outputs/eval/sample_ch/shard_7.jsonl": { "rows": 100, "candidates": 1600 } }, "unsolved_keys": [ [ 2, 3, 5, 12 ], [ 3, 5, 7, 13 ], [ 2, 9, 13, 13 ], [ 2, 4, 10, 10 ], [ 1, 3, 4, 6 ], [ 2, 7, 7, 10 ], [ 1, 4, 5, 6 ], [ 7, 8, 8, 13 ], [ 1, 6, 6, 8 ], [ 3, 3, 8, 8 ], [ 6, 9, 9, 10 ], [ 4, 7, 11, 13 ], [ 1, 6, 11, 13 ], [ 2, 5, 6, 9 ], [ 2, 2, 6, 9 ], [ 2, 2, 7, 10 ], [ 4, 4, 7, 13 ], [ 8, 8, 9, 12 ], [ 6, 6, 7, 11 ], [ 8, 9, 11, 11 ], [ 9, 11, 12, 12 ] ], "reason_counts": { "value is 48, expected 24": 1110, "ok": 958, "value is 30, expected 24": 894, "value is 12, expected 24": 891, "value is 40, expected 24": 870, "value is 18, expected 24": 819, "value is 60, expected 24": 790, "value is 20, expected 24": 766, "value is 0, expected 24": 765, "value is 16, expected 24": 760, "value is 36, expected 24": 752, "value is 72, expected 24": 637, "value is 6, expected 24": 628, "value is 84, expected 24": 523, "value is 22, expected 24": 509, "empty expression": 503, "numbers are not used exactly once": 495, "value is 21, expected 24": 487, "value is 25, expected 24": 471, "value is 8, expected 24": 471, "value is 32, expected 24": 452, "value is 42, expected 24": 440, "value is 26, expected 24": 399, "value is 28, expected 24": 394, "value is 27, expected 24": 360, "value is 14, expected 24": 343, "value is 96, expected 24": 339, "value is 10, expected 24": 308, "value is 15, expected 24": 298, "value is 33, expected 24": 293, "value is 35, expected 24": 260, "value is 23, expected 24": 246, "value is 9, expected 24": 226, "value is 54, expected 24": 218, "value is 56, expected 24": 217, "value is 19, expected 24": 216, "value is 29, expected 24": 216, "value is 66, expected 24": 166, "value is 4, expected 24": 161, "value is 31, expected 24": 147, "value is 34, expected 24": 145, "value is 37, expected 24": 140, "value is 50, expected 24": 132, "value is 39, expected 24": 122, "value is 45, expected 24": 120, "value is 120, expected 24": 120, "value is -12, expected 24": 117, "value is 44, expected 24": 112, "value is 64, expected 24": 106, "value is 52, expected 24": 105, "value is 13, expected 24": 103, "value is 90, expected 24": 102, "contains disallowed characters": 98, "value is 70, expected 24": 97, "value is 9.33333333333, expected 24": 96, "value is 43, expected 24": 95, "value is 80, expected 24": 91, "value is 38, expected 24": 88, "value is 7, expected 24": 76, "value is 108, expected 24": 76, "value is 17, expected 24": 75, "value is 16.6666666667, expected 24": 75, "value is 16.5, expected 24": 74, "value is 10.5, expected 24": 72, "value is -16, expected 24": 70, "value is 6.4, expected 24": 63, "value is 18.6666666667, expected 24": 59, "value is 3, expected 24": 57, "value is 77, expected 24": 57, "value is -6, expected 24": 53, "value is 5, expected 24": 53, "value is 78, expected 24": 51, "value is 20.5714285714, expected 24": 49, "value is -7, expected 24": 47, "value is 21.3333333333, expected 24": 46, "value is 49, expected 24": 45, "value is 104, expected 24": 45, "value is 22.5, expected 24": 43, "value is 12.5, expected 24": 41, "value is 91, expected 24": 41, "value is 11, expected 24": 40, "value is 98, expected 24": 38, "value is 65, expected 24": 38, "value is -36, expected 24": 38, "value is -8, expected 24": 37, "value is 5.6, expected 24": 34, "value is 88, expected 24": 34, "value is -22, expected 24": 34, "value is 26.4, expected 24": 33, "value is 63, expected 24": 32, "value is 33.3333333333, expected 24": 31, "value is 25.6, expected 24": 27, "value is 5.71428571429, expected 24": 26, "value is 4.66666666667, expected 24": 26, "value is 168, expected 24": 26, "value is 82.5, expected 24": 26, "value is 12.8571428571, expected 24": 24, "value is -136, expected 24": 24, "value is 17.6, expected 24": 23, "value is 24.2, expected 24": 23, "value is 25.6666666667, expected 24": 23, "value is 32.5714285714, expected 24": 23, "value is 52.25, expected 24": 21, "value is -26, expected 24": 20, "value is 17.3333333333, expected 24": 20, "value is 6.5, expected 24": 20, "value is 46, expected 24": 19, "value is 140, expected 24": 19, "value is 55, expected 24": 19, "value is 160, expected 24": 19, "value is 13.5, expected 24": 19, "value is 41, expected 24": 18, "value is 51.0769230769, expected 24": 18, "value is 6.66666666667, expected 24": 18, "value is 10.6666666667, expected 24": 18, "value is 14.4, expected 24": 17, "value is 9.6, expected 24": 17, "value is 110, expected 24": 16, "value is 71, expected 24": 16, "value is 11.2, expected 24": 16, "value is 126, expected 24": 16, "value is 27.1428571429, expected 24": 15, "value is 70.5, expected 24": 15, "value is 21.5, expected 24": 14, "value is 4.5, expected 24": 14, "value is 13.3333333333, expected 24": 14, "value is 28.8, expected 24": 14, "value is 25.6923076923, expected 24": 14, "value is 25.5, expected 24": 13, "value is 22.4, expected 24": 13, "value is 24.6153846154, expected 24": 13, "value is 29.3333333333, expected 24": 13, "value is 240, expected 24": 13, "value is 228, expected 24": 13, "value is 10.8, expected 24": 13, "value is 21.6363636364, expected 24": 13, "value is 15.4285714286, expected 24": 13, "value is 93, expected 24": 13, "value is 57, expected 24": 12, "value is 61, expected 24": 12, "value is 65.5, expected 24": 12, "value is 50.6666666667, expected 24": 12, "value is -20, expected 24": 11, "value is 132, expected 24": 11, "division by zero": 11, "value is 144, expected 24": 11, "value is 81, expected 24": 11, "value is 69.3333333333, expected 24": 11, "value is 47, expected 24": 10, "value is 24.5, expected 24": 10, "value is -9, expected 24": 10, "value is 23.5714285714, expected 24": 10, "value is 22.8571428571, expected 24": 10, "value is 68, expected 24": 10, "value is 58.5, expected 24": 10, "value is 5.14285714286, expected 24": 10, "value is 12.6666666667, expected 24": 10, "value is 8.33333333333, expected 24": 10, "value is 7.61538461538, expected 24": 9, "value is 37.3333333333, expected 24": 9, "value is 4.36363636364, expected 24": 9, "value is 19.5, expected 24": 9, "value is 13.7142857143, expected 24": 9, "value is 8.125, expected 24": 9, "value is 150, expected 24": 9, "value is -24, expected 24": 9, "value is 40.5, expected 24": 9, "value is 17.8, expected 24": 9, "value is 5.4, expected 24": 9, "value is 186, expected 24": 9, "value is 22.2857142857, expected 24": 8, "value is 105, expected 24": 8, "value is 136, expected 24": 8, "value is 26.5, expected 24": 8, "value is 8.66666666667, expected 24": 8, "value is 22.6666666667, expected 24": 8, "value is 6.54545454545, expected 24": 8, "value is 8.25, expected 24": 8, "value is 19.7142857143, expected 24": 8, "value is 31.6666666667, expected 24": 8, "value is 35.6, expected 24": 8, "value is 28.3636363636, expected 24": 8, "value is 67, expected 24": 8, "value is 9.5, expected 24": 7, "value is 117, expected 24": 7, "value is 8.4, expected 24": 7, "value is 31.5, expected 24": 7, "value is 44.5714285714, expected 24": 7, "value is 2, expected 24": 7, "value is 18.3333333333, expected 24": 7, "numbers must be positive integers": 7, "value is 23.2727272727, expected 24": 7, "value is 11.6666666667, expected 24": 7, "value is 89.1428571429, expected 24": 7, "value is 24.7272727273, expected 24": 7, "value is 13.6, expected 24": 7, "value is 23.1111111111, expected 24": 6, "value is 51.3333333333, expected 24": 6, "value is 53, expected 24": 6, "value is 38.5, expected 24": 6, "value is 23.7777777778, expected 24": 6, "value is 112, expected 24": 6, "value is 44.625, expected 24": 6, "value is 47.2, expected 24": 6, "value is 21.6, expected 24": 6, "value is -56, expected 24": 6, "value is 23.6, expected 24": 6, "value is -18, expected 24": 6, "value is 11.5384615385, expected 24": 6, "value is 17.4, expected 24": 6, "value is 25.3333333333, expected 24": 6, "value is 23.25, expected 24": 6, "value is 17.7777777778, expected 24": 6, "value is 35.1, expected 24": 6, "value is 12.7272727273, expected 24": 6, "value is 8.55555555556, expected 24": 6, "value is 16.2, expected 24": 6, "value is 22.25, expected 24": 6, "value is 75, expected 24": 5, "value is -15, expected 24": 5, "value is -65, expected 24": 5, "value is 9.42857142857, expected 24": 5, "value is 210, expected 24": 5, "value is 19.3846153846, expected 24": 5, "value is -28, expected 24": 5, "value is 23.3333333333, expected 24": 5, "value is 9.23076923077, expected 24": 5, "value is 22.4285714286, expected 24": 5, "value is 35.2, expected 24": 5, "value is 20.25, expected 24": 5, "value is 22.2, expected 24": 5, "value is 22.6, expected 24": 5, "value is 21.8333333333, expected 24": 5, "value is 180, expected 24": 5, "value is 21.6666666667, expected 24": 5, "value is 33.6666666667, expected 24": 5, "value is 40.4285714286, expected 24": 5, "value is 7.33333333333, expected 24": 5, "value is 9.81818181818, expected 24": 5, "value is 38.6666666667, expected 24": 5, "value is 130, expected 24": 5, "value is 14.6666666667, expected 24": 5, "value is 16.3636363636, expected 24": 5, "value is 28.5, expected 24": 4, "value is 7.71428571429, expected 24": 4, "value is 5.77777777778, expected 24": 4, "value is 51, expected 24": 4, "value is 11.4285714286, expected 24": 4, "value is 59, expected 24": 4, "value is 83, expected 24": 4, "value is 22.75, expected 24": 4, "value is 22.1428571429, expected 24": 4, "value is 19.8, expected 24": 4, "value is 40.8, expected 24": 4, "value is 25.0909090909, expected 24": 4, "value is 27.4, expected 24": 4, "value is 32.4, expected 24": 4, "value is 15.25, expected 24": 4, "value is 25.7142857143, expected 24": 4, "value is -9.6, expected 24": 4, "value is 26.6666666667, expected 24": 4, "value is 243, expected 24": 4, "value is 39.1111111111, expected 24": 4, "value is 99, expected 24": 4, "value is 56.5714285714, expected 24": 4, "value is 17.4545454545, expected 24": 4, "value is 5.2, expected 24": 4, "value is 85.3333333333, expected 24": 4, "value is -91, expected 24": 4, "value is 114, expected 24": 3, "value is 27.5, expected 24": 3, "value is 37.5, expected 24": 3, "value is 13.2, expected 24": 3, "value is 92, expected 24": 3, "value is 19.6, expected 24": 3, "value is 7.14285714286, expected 24": 3, "value is 20.5, expected 24": 3, "value is 6.85714285714, expected 24": 3, "value is 7.38461538462, expected 24": 3, "value is 49.1428571429, expected 24": 3, "value is 48.7272727273, expected 24": 3, "value is 72.2857142857, expected 24": 3, "value is 48.3, expected 24": 3, "value is 121, expected 24": 3, "value is 23.5, expected 24": 3, "value is 22.7692307692, expected 24": 3, "value is 47.6666666667, expected 24": 3, "value is 8.61538461538, expected 24": 3, "value is 22.1666666667, expected 24": 3, "value is 19.6363636364, expected 24": 3, "value is 74.7692307692, expected 24": 3, "value is 68.4, expected 24": 3, "value is 4.14285714286, expected 24": 3, "value is 4.91666666667, expected 24": 3, "value is 76, expected 24": 3, "value is 45.3333333333, expected 24": 3, "value is 138, expected 24": 3, "value is 8.27272727273, expected 24": 3, "value is 11.25, expected 24": 3, "value is 19.3333333333, expected 24": 3, "value is 6.46153846154, expected 24": 3, "value is 23.1428571429, expected 24": 3, "value is -48, expected 24": 3, "value is -2, expected 24": 3, "value is 36.3636363636, expected 24": 3, "value is -30, expected 24": 3, "value is 9.71428571429, expected 24": 3, "value is 8.16666666667, expected 24": 3, "value is 7.09090909091, expected 24": 3, "value is 14.5714285714, expected 24": 3, "unsupported expression node: UnaryOp": 3, "value is 36.4, expected 24": 3, "value is 106, expected 24": 3, "value is 34.6666666667, expected 24": 2, "value is 182, expected 24": 2, "value is 19.2, expected 24": 2, "value is 5.45454545455, expected 24": 2, "value is 32.8, expected 24": 2, "invalid syntax (, line 1)": 2, "value is -42, expected 24": 2, "value is 73, expected 24": 2, "value is 33.4285714286, expected 24": 2, "value is 33.6, expected 24": 2, "value is 8.8, expected 24": 2, "value is 2.75, expected 24": 2, "value is 17.875, expected 24": 2, "value is 5.5, expected 24": 2, "value is 20.4285714286, expected 24": 2, "value is 22.5714285714, expected 24": 2, "value is 15.5, expected 24": 2, "value is 62, expected 24": 2, "value is 50.5, expected 24": 2, "value is 29.5, expected 24": 2, "value is 28.1666666667, expected 24": 2, "value is 7.27272727273, expected 24": 2, "value is 18.5, expected 24": 2, "value is 30.1428571429, expected 24": 2, "value is 100, expected 24": 2, "value is 40.8571428571, expected 24": 2, "value is 15.2, expected 24": 2, "value is 7.63636363636, expected 24": 2, "value is 42.3076923077, expected 24": 2, "value is 1.28571428571, expected 24": 2, "value is 264, expected 24": 2, "value is 62.4, expected 24": 2, "value is 38.4, expected 24": 2, "value is 7.5, expected 24": 2, "value is 59.5, expected 24": 2, "value is 33.5, expected 24": 2, "value is 24.6666666667, expected 24": 2, "value is 22.6153846154, expected 24": 2, "value is 15.1666666667, expected 24": 2, "value is 15.3333333333, expected 24": 2, "value is 13.6363636364, expected 24": 2, "value is 6.92307692308, expected 24": 2, "value is 4.25, expected 24": 2, "value is 18.8571428571, expected 24": 2, "value is 53.8888888889, expected 24": 2, "value is 41.8181818182, expected 24": 2, "value is 24.4, expected 24": 2, "value is 116, expected 24": 2, "value is 5.90909090909, expected 24": 2, "value is 7.42857142857, expected 24": 2, "value is -4.5, expected 24": 2, "value is 8.75, expected 24": 2, "value is 26.8571428571, expected 24": 2, "value is 29.7142857143, expected 24": 2, "value is 1, expected 24": 2, "value is 8.44444444444, expected 24": 2, "value is 66.6666666667, expected 24": 2, "value is -2.57142857143, expected 24": 2, "value is 34.5555555556, expected 24": 2, "value is 15.75, expected 24": 2, "value is 52.5, expected 24": 2, "value is -14, expected 24": 2, "value is 17.5, expected 24": 2, "value is 24.4615384615, expected 24": 2, "value is 25.3846153846, expected 24": 2, "value is 23.4545454545, expected 24": 2, "value is 19.5555555556, expected 24": 2, "value is 53.1666666667, expected 24": 2, "value is 24.75, expected 24": 2, "value is 33.2, expected 24": 2, "value is 7.58333333333, expected 24": 2, "value is 58.1538461538, expected 24": 2, "value is 14.7777777778, expected 24": 2, "value is 1164, expected 24": 2, "value is 14.5, expected 24": 1, "value is 156, expected 24": 1, "value is 252, expected 24": 1, "value is 125, expected 24": 1, "value is 10.4, expected 24": 1, "value is 174, expected 24": 1, "value is 258, expected 24": 1, "value is 4.85714285714, expected 24": 1, "value is 10.6363636364, expected 24": 1, "value is 1.44444444444, expected 24": 1, "value is 37.7142857143, expected 24": 1, "value is 85, expected 24": 1, "value is 440, expected 24": 1, "value is 23.4285714286, expected 24": 1, "value is 2.18181818182, expected 24": 1, "value is 1.85714285714, expected 24": 1, "value is 4.90909090909, expected 24": 1, "value is 19.4, expected 24": 1, "value is 25.2, expected 24": 1, "value is 11.1428571429, expected 24": 1, "value is -49, expected 24": 1, "value is 115.909090909, expected 24": 1, "value is -2.69230769231, expected 24": 1, "value is 24.1818181818, expected 24": 1, "value is -84, expected 24": 1, "value is 74, expected 24": 1, "value is 53.3333333333, expected 24": 1, "value is 62.1428571429, expected 24": 1, "value is 0.674698795181, expected 24": 1, "value is 95, expected 24": 1, "value is 8.72727272727, expected 24": 1, "value is 242, expected 24": 1, "value is -3.5, expected 24": 1, "value is 79.3333333333, expected 24": 1, "value is 5.625, expected 24": 1, "value is 184, expected 24": 1, "value is 4.125, expected 24": 1, "value is 21.8, expected 24": 1, "value is -171, expected 24": 1, "value is 35.8333333333, expected 24": 1, "value is 51.2, expected 24": 1, "value is -243, expected 24": 1, "value is 29.8, expected 24": 1, "value is 53.5714285714, expected 24": 1, "value is 8.57142857143, expected 24": 1, "value is -72, expected 24": 1, "value is 13.0909090909, expected 24": 1, "value is 350, expected 24": 1, "value is 26.8, expected 24": 1, "value is 17.2, expected 24": 1, "value is 94.4, expected 24": 1, "value is 55.25, expected 24": 1, "value is 2.15384615385, expected 24": 1, "value is 30.5714285714, expected 24": 1, "value is 24.7142857143, expected 24": 1, "value is 17.5384615385, expected 24": 1, "value is -66, expected 24": 1, "value is 16.6153846154, expected 24": 1, "value is 11.0769230769, expected 24": 1, "value is -5, expected 24": 1, "value is 27.25, expected 24": 1, "value is -32, expected 24": 1, "value is 21.25, expected 24": 1, "value is 6.25, expected 24": 1, "value is 9.375, expected 24": 1, "value is 4.61538461538, expected 24": 1, "value is 7.69230769231, expected 24": 1, "value is 43.5, expected 24": 1, "value is 6.16666666667, expected 24": 1, "value is 25.8, expected 24": 1, "value is 18.4285714286, expected 24": 1, "value is 21.8571428571, expected 24": 1, "value is 16.2222222222, expected 24": 1, "value is 24.6, expected 24": 1, "value is 4.4, expected 24": 1, "value is 1.45454545455, expected 24": 1, "value is 165, expected 24": 1, "value is 220, expected 24": 1, "value is 300, expected 24": 1, "value is 19.6666666667, expected 24": 1, "value is 20.3333333333, expected 24": 1, "value is 27.3333333333, expected 24": 1, "value is 9.75, expected 24": 1, "value is 46.2857142857, expected 24": 1, "value is -50, expected 24": 1, "value is 4.44444444444, expected 24": 1, "value is 94, expected 24": 1, "value is 48.9, expected 24": 1, "value is 97.8, expected 24": 1, "value is 162, expected 24": 1, "value is 247, expected 24": 1, "value is 175, expected 24": 1, "value is 29.6666666667, expected 24": 1, "value is 217, expected 24": 1, "value is 64.1666666667, expected 24": 1, "value is 20.4, expected 24": 1, "value is -26.6666666667, expected 24": 1, "value is 36.8181818182, expected 24": 1, "value is 22.8, expected 24": 1, "value is 45.1, expected 24": 1, "value is 24.25, expected 24": 1, "value is 9.77777777778, expected 24": 1, "value is 12.8, expected 24": 1, "value is 23.625, expected 24": 1, "value is -2.12903225806, expected 24": 1, "value is 141.166666667, expected 24": 1, "value is 23.8333333333, expected 24": 1, "value is 80.5714285714, expected 24": 1, "value is 54.6666666667, expected 24": 1, "value is -14.4, expected 24": 1, "value is 74.6666666667, expected 24": 1, "value is 29.1428571429, expected 24": 1, "value is 46.6666666667, expected 24": 1, "value is 61.9090909091, expected 24": 1, "value is 4.42857142857, expected 24": 1, "value is 74.1818181818, expected 24": 1, "value is 26.75, expected 24": 1, "value is -3, expected 24": 1, "value is -52, expected 24": 1, "value is 17.1428571429, expected 24": 1, "value is 12.6, expected 24": 1, "value is -2.55555555556, expected 24": 1, "value is 14.8571428571, expected 24": 1, "value is 29.25, expected 24": 1, "value is 27.75, expected 24": 1, "value is 6.6, expected 24": 1, "value is 2.4, expected 24": 1, "value is -10, expected 24": 1, "value is 4.58333333333, expected 24": 1, "value is 0.96, expected 24": 1, "value is 17.1111111111, expected 24": 1, "value is 20.2222222222, expected 24": 1, "value is 19.4444444444, expected 24": 1, "value is 69, expected 24": 1, "value is 38.7272727273, expected 24": 1, "value is 29.1666666667, expected 24": 1, "value is 5.83333333333, expected 24": 1, "value is 23.2222222222, expected 24": 1, "value is 23.375, expected 24": 1, "value is 62.5, expected 24": 1, "value is 80.5, expected 24": 1, "value is 5.38461538462, expected 24": 1, "value is 32.5, expected 24": 1, "value is 1.05405405405, expected 24": 1, "value is 81.5, expected 24": 1, "value is 46.8, expected 24": 1, "value is 135, expected 24": 1, "value is 61.2, expected 24": 1, "value is 22.9090909091, expected 24": 1, "value is 24.3333333333, expected 24": 1, "value is 30.6666666667, expected 24": 1 } }