{ "count": 100, "solved": 18, "solved_rate": 0.18, "solved_at_n_count": 18, "solved_at_n_rate": 0.18, "pass_at_1_count": 1, "pass_at_1_rate": 0.01, "candidate_count_total": 1600, "candidate_count_mean_per_problem": 16, "num_samples_max": 16, "num_samples_min": 16, "reason_counts": { "numbers are not used exactly once": 614, "ok": 31, "numbers must be positive integers": 26, "contains disallowed characters": 10, "value is 840, expected 62": 10, "value is 88, expected 26": 10, "value is 36, expected 20": 9, "value is 16.2608695652, expected 33": 9, "empty expression": 8, "value is 222, expected 88": 7, "value is 496, expected 100": 7, "value is 777, expected 59": 7, "unsupported expression node: UnaryOp": 6, "value is 23.0322580645, expected 44": 6, "value is 350, expected 26": 6, "value is 1600, expected 100": 6, "value is 120, expected 32": 6, "value is 310, expected 41": 6, "value is 992, expected 60": 6, "value is 830, expected 77": 5, "value is 19.4520547945, expected 44": 5, "value is 99, expected 30": 5, "value is 3572, expected 97": 5, "value is 79, expected 17": 5, "value is 1664, expected 96": 5, "value is 715, expected 78": 5, "value is 56, expected 22": 5, "value is 21, expected 38": 5, "value is 102, expected 37": 5, "value is 392, expected 45": 5, "value is 15, expected 66": 5, "value is 133, expected 77": 5, "value is 483, expected 62": 4, "value is 106, expected 76": 4, "value is 94.08, expected 24": 4, "value is 47.2131147541, expected 90": 4, "value is 895, expected 75": 4, "value is 32, expected 36": 4, "value is 924, expected 65": 4, "value is 960, expected 80": 4, "value is 218.75, expected 61": 4, "value is 129, expected 65": 4, "value is 36, expected 37": 4, "value is 259, expected 47": 4, "value is 798, expected 43": 4, "value is 14, expected 15": 4, "value is 393, expected 71": 4, "value is 324, expected 36": 4, "value is 131, expected 14": 4, "value is 638, expected 83": 4, "value is 727, expected 87": 4, "value is 300, expected 87": 4, "value is 1900, expected 51": 4, "value is 324, expected 92": 4, "value is 17.4025974026, expected 44": 3, "value is 272, expected 17": 3, "value is 44.9770114943, expected 96": 3, "value is 27, expected 28": 3, "value is 3956, expected 76": 3, "value is 74.170212766, expected 48": 3, "value is 595, expected 100": 3, "value is 703, expected 97": 3, "value is 1216, expected 17": 3, "value is 30, expected 26": 3, "value is 3.42857142857, expected 52": 3, "value is 61, expected 75": 3, "value is 1596, expected 75": 3, "value is 1943, expected 96": 3, "value is 156, expected 98": 3, "value is 888, expected 13": 3, "value is 241, expected 94": 3, "value is 675, expected 83": 3, "value is 96.28, expected 53": 3, "value is 99, expected 80": 3, "value is 243, expected 65": 3, "value is 255.461538462, expected 65": 3, "value is 26.6666666667, expected 58": 3, "value is 501, expected 83": 3, "value is 437, expected 45": 3, "value is 390, expected 43": 3, "value is 2255, expected 96": 3, "value is 144, expected 14": 3, "value is 359, expected 14": 3, "value is 21.0526315789, expected 24": 3, "value is 143, expected 35": 3, "value is 129, expected 35": 3, "value is 154, expected 66": 3, "value is 371, expected 60": 3, "value is 440, expected 15": 3, "value is 154, expected 29": 3, "value is 68, expected 22": 3, "value is 153, expected 53": 3, "value is 96, expected 78": 3, "value is 11.7049180328, expected 58": 3, "value is 46.6363636364, expected 51": 3, "value is 426, expected 92": 3, "value is 964, expected 77": 2, "value is 4.12987012987, expected 44": 2, "value is 11658, expected 96": 2, "value is 139, expected 28": 2, "value is 26.5, expected 76": 2, "value is 93.4375, expected 24": 2, "value is 155.52, expected 24": 2, "value is 46.8, expected 90": 2, "value is 92, expected 48": 2, "value is 14774, expected 48": 2, "value is -8.93617021277, expected 100": 2, "value is 505, expected 100": 2, "value is 1748, expected 44": 2, "value is 20.8387096774, expected 44": 2, "value is 186, expected 28": 2, "value is 151, expected 98": 2, "value is 1320, expected 92": 2, "value is 26.1044776119, expected 92": 2, "value is 229.210526316, expected 92": 2, "value is 133, expected 94": 2, "value is 224, expected 94": 2, "value is -82, expected 94": 2, "value is 63, expected 28": 2, "value is 3.96907216495, expected 28": 2, "value is 8.87628865979, expected 28": 2, "value is 1080, expected 78": 2, "value is 1680, expected 83": 2, "value is 1722, expected 83": 2, "value is 642, expected 53": 2, "value is 1190, expected 53": 2, "value is -225, expected 50": 2, "value is 225, expected 50": 2, "value is 819, expected 50": 2, "value is 131.2, expected 14": 2, "value is 73.2631578947, expected 90": 2, "value is 268.444444444, expected 90": 2, "value is 69.9090909091, expected 38": 2, "value is 1857.77777778, expected 38": 2, "value is 440, expected 61": 2, "value is -1110, expected 65": 2, "value is 774, expected 77": 2, "value is 87.3658536585, expected 58": 2, "value is 308, expected 58": 2, "value is 170, expected 91": 2, "value is 70, expected 100": 2, "value is 1426, expected 100": 2, "value is 1750, expected 36": 2, "value is 121.411111111, expected 36": 2, "value is 34.2, expected 36": 2, "value is 16.7368421053, expected 83": 2, "value is 84, expected 47": 2, "value is 111, expected 47": 2, "value is 108, expected 59": 2, "value is 99, expected 59": 2, "value is 1128, expected 15": 2, "value is 1209, expected 15": 2, "value is 2229, expected 71": 2, "value is 722, expected 57": 2, "value is 59.5833333333, expected 30": 2, "value is 571, expected 30": 2, "value is 27.3962264151, expected 30": 2, "value is 782, expected 11": 2, "value is 4480, expected 24": 2, "value is 59840, expected 24": 2, "value is 9.35, expected 24": 2, "value is 52.1276595745, expected 26": 2, "value is -560, expected 26": 2, "value is 28.085106383, expected 26": 2, "value is 32.1489361702, expected 26": 2, "value is 116, expected 35": 2, "value is 85, expected 86": 2, "value is 1081, expected 22": 2, "value is 121.268656716, expected 83": 2, "value is 1798, expected 83": 2, "value is 630, expected 72": 2, "value is 685.076923077, expected 72": 2, "value is 211, expected 78": 2, "value is 175.058823529, expected 78": 2, "value is 5451, expected 87": 2, "value is 9.42391304348, expected 58": 2, "value is 52.4262295082, expected 58": 2, "value is 5.61224489796, expected 51": 2, "value is -32, expected 88": 1, "division by zero": 1, "value is 12.0833333333, expected 72": 1, "value is 1072, expected 77": 1, "value is -1410, expected 77": 1, "value is 17.2307692308, expected 77": 1, "value is 98.6268656716, expected 77": 1, "value is 1276, expected 77": 1, "value is 78.5820895522, expected 77": 1, "value is -5, expected 77": 1, "value is 45, expected 44": 1, "value is 4.68493150685, expected 44": 1, "value is 1203.33333333, expected 44": 1, "value is 40, expected 44": 1, "value is -13, expected 17": 1, "value is 64, expected 17": 1, "value is 40.5517241379, expected 96": 1, "value is 160.034482759, expected 96": 1, "value is 7654, expected 96": 1, "value is 41.1098901099, expected 96": 1, "value is 172, expected 96": 1, "value is 4998, expected 96": 1, "value is 129.612244898, expected 96": 1, "value is 151.954022989, expected 96": 1, "value is 396, expected 96": 1, "value is -236.833333333, expected 96": 1, "value is 4176, expected 96": 1, "value is 224, expected 20": 1, "value is 56.5925925926, expected 66": 1, "value is 89, expected 66": 1, "value is 219.333333333, expected 30": 1, "value is 880, expected 62": 1, "value is 6.27906976744, expected 76": 1, "value is 30.2816901408, expected 76": 1, "value is 132, expected 76": 1, "value is 590, expected 76": 1, "value is 1904, expected 76": 1, "value is 182, expected 76": 1, "value is 23.8604651163, expected 76": 1, "value is 1152, expected 84": 1, "value is 6429, expected 84": 1, "value is 632.888888889, expected 55": 1, "value is -208, expected 55": 1, "value is 163, expected 55": 1, "value is -260, expected 55": 1, "value is 665, expected 55": 1, "value is 243, expected 55": 1, "value is 12.3076923077, expected 24": 1, "value is 166.4, expected 24": 1, "value is 33.5208333333, expected 24": 1, "value is 5.66153846154, expected 24": 1, "value is -1098, expected 90": 1, "value is 270, expected 90": 1, "value is 12.4590163934, expected 90": 1, "value is -90, expected 90": 1, "value is 928, expected 90": 1, "value is 2897.5, expected 90": 1, "value is 115.9, expected 90": 1, "value is 14960, expected 48": 1, "value is 934, expected 48": 1, "value is 96, expected 48": 1, "value is 186, expected 48": 1, "value is 90.8117647059, expected 48": 1, "value is 95.1325301205, expected 48": 1, "value is 1020, expected 48": 1, "value is 74.8363636364, expected 100": 1, "value is 856, expected 100": 1, "value is 4061, expected 100": 1, "value is 1000, expected 100": 1, "value is 111.904761905, expected 100": 1, "value is -934, expected 100": 1, "value is 72.8, expected 100": 1, "value is 989, expected 97": 1, "value is 2304, expected 17": 1, "value is 14.1578947368, expected 17": 1, "value is 2356, expected 17": 1, "value is 59.7777777778, expected 17": 1, "value is 128.710526316, expected 17": 1, "value is 213, expected 17": 1, "value is 17.7631578947, expected 17": 1, "value is 70, expected 17": 1, "value is 84, expected 44": 1, "value is 113.884615385, expected 44": 1, "value is 1739, expected 44": 1, "value is 513, expected 44": 1, "value is 9.14285714286, expected 44": 1, "value is 196.923076923, expected 44": 1, "value is 1323, expected 44": 1, "value is 448, expected 44": 1, "value is 36, expected 26": 1, "value is 15.2307692308, expected 26": 1, "value is 104, expected 26": 1, "value is -69.2941176471, expected 44": 1, "value is 255, expected 44": 1, "value is 192, expected 44": 1, "value is 438, expected 44": 1, "value is 595, expected 44": 1, "value is 54, expected 52": 1, "value is 120.545454545, expected 52": 1, "value is -36, expected 52": 1, "value is 36, expected 52": 1, "value is 6418.72727273, expected 52": 1, "value is 59.476744186, expected 52": 1, "value is 2753.1025641, expected 52": 1, "value is 880, expected 52": 1, "value is 53, expected 52": 1, "value is 152.393939394, expected 52": 1, "value is 899, expected 75": 1, "value is 64.8, expected 75": 1, "value is 395, expected 75": 1, "value is -7, expected 75": 1, "value is -210, expected 28": 1, "value is 67, expected 28": 1, "value is 128, expected 28": 1, "value is 576, expected 28": 1, "value is 6.11111111111, expected 28": 1, "value is 276, expected 28": 1, "value is 192, expected 28": 1, "value is -366, expected 28": 1, "value is 8.10752688172, expected 96": 1, "value is 279, expected 96": 1, "value is 184, expected 98": 1, "value is 2013, expected 98": 1, "value is 56, expected 98": 1, "value is 406, expected 98": 1, "value is 48, expected 98": 1, "value is 2822, expected 63": 1, "value is 123.5, expected 13": 1, "value is 76.6153846154, expected 13": 1, "value is 31.4776119403, expected 92": 1, "value is 1512, expected 92": 1, "value is 262, expected 92": 1, "value is 73, expected 92": 1, "value is -5313, expected 92": 1, "value is 7.01754385965, expected 92": 1, "value is 5300, expected 92": 1, "value is 318.142857143, expected 94": 1, "value is 1981, expected 94": 1, "value is 2040, expected 94": 1, "value is -70.2857142857, expected 94": 1, "value is 2255, expected 94": 1, "value is 492, expected 94": 1, "value is 1009.5, expected 28": 1, "value is 294, expected 28": 1, "value is 588, expected 28": 1, "value is 19.8453608247, expected 28": 1, "value is 1064, expected 28": 1, "value is 1655.92857143, expected 28": 1, "value is 7.44117647059, expected 33": 1, "value is 2714, expected 33": 1, "value is 760, expected 33": 1, "value is 702, expected 33": 1, "value is 57, expected 24": 1, "value is 2.12903225806, expected 24": 1, "value is 1763, expected 83": 1, "value is 132, expected 83": 1, "value is 15.9104477612, expected 83": 1, "value is 1056, expected 83": 1, "value is -1, expected 83": 1, "value is 0, expected 83": 1, "value is 26.1463414634, expected 83": 1, "value is 161.28, expected 53": 1, "value is 700, expected 53": 1, "value is 138, expected 53": 1, "value is 890.384615385, expected 53": 1, "value is 78.8461538462, expected 53": 1, "value is 44.7529411765, expected 65": 1, "value is 1700, expected 65": 1, "value is 1044, expected 50": 1, "value is 26.3157894737, expected 14": 1, "value is 173.684210526, expected 14": 1, "value is 855, expected 14": 1, "value is 441.666666667, expected 14": 1, "value is 95, expected 14": 1, "value is 2160, expected 14": 1, "value is 4.5, expected 14": 1, "value is 357.894736842, expected 14": 1, "value is 146.4, expected 14": 1, "value is 339, expected 80": 1, "value is 861, expected 80": 1, "value is 135.789473684, expected 90": 1, "value is 1800, expected 90": 1, "value is 296.8, expected 90": 1, "value is 86.1578947368, expected 90": 1, "value is 27.7894736842, expected 90": 1, "value is 880, expected 38": 1, "value is 18.5454545455, expected 38": 1, "value is 65.0833333333, expected 38": 1, "value is 1012, expected 38": 1, "value is 456, expected 19": 1, "value is 87.4, expected 19": 1, "value is 626.24, expected 61": 1, "value is 130, expected 61": 1, "value is 107.666666667, expected 61": 1, "value is 1650, expected 61": 1, "value is 20.52, expected 61": 1, "value is 91, expected 65": 1, "value is 49.5, expected 65": 1, "value is 11088, expected 65": 1, "value is 252, expected 65": 1, "value is 950, expected 77": 1, "value is 42.7826086957, expected 58": 1, "value is 1088, expected 58": 1, "value is 74, expected 58": 1, "value is 40, expected 58": 1, "value is -0.512195121951, expected 58": 1, "value is 1020, expected 37": 1, "value is 1100, expected 37": 1, "value is 33.35, expected 37": 1, "value is 110, expected 37": 1, "value is 3, expected 37": 1, "value is 216, expected 37": 1, "value is -36, expected 37": 1, "value is 7224, expected 91": 1, "value is 47.4, expected 100": 1, "value is 99, expected 100": 1, "value is 760, expected 36": 1, "value is 37.358490566, expected 36": 1, "value is 814, expected 36": 1, "value is 29.4444444444, expected 36": 1, "value is 2660, expected 36": 1, "value is 2610, expected 36": 1, "value is 123.2, expected 36": 1, "value is 342, expected 36": 1, "value is 54.3396226415, expected 36": 1, "value is 755, expected 83": 1, "value is 473.2, expected 83": 1, "value is 12, expected 83": 1, "value is 3648, expected 83": 1, "value is 519, expected 83": 1, "value is 321, expected 83": 1, "value is 10.8791208791, expected 83": 1, "value is 175, expected 47": 1, "value is 64, expected 47": 1, "value is 44, expected 47": 1, "value is 42, expected 45": 1, "value is 43, expected 32": 1, "value is 252, expected 32": 1, "invalid syntax (, line 1)": 1, "value is -99, expected 59": 1, "value is 333, expected 59": 1, "value is 4503, expected 62": 1, "value is 59, expected 43": 1, "value is 196, expected 96": 1, "value is 1584, expected 15": 1, "value is -1584, expected 15": 1, "value is 107.925925926, expected 15": 1, "value is 891, expected 15": 1, "value is 672, expected 15": 1, "value is -67.5, expected 15": 1, "value is 107.555555556, expected 15": 1, "value is 18.5, expected 15": 1, "value is 103.905405405, expected 71": 1, "value is 37.4594594595, expected 71": 1, "value is 95.9259259259, expected 71": 1, "value is 3807, expected 71": 1, "value is -26.1363636364, expected 71": 1, "value is 407, expected 24": 1, "value is 3.53658536585, expected 14": 1, "value is 4717, expected 36": 1, "value is 5041, expected 36": 1, "value is 56, expected 29": 1, "value is 851, expected 29": 1, "value is 61.3142857143, expected 29": 1, "value is 49, expected 14": 1, "value is 86, expected 14": 1, "value is -32, expected 14": 1, "value is 420, expected 14": 1, "value is 4332, expected 57": 1, "value is -0.5, expected 57": 1, "value is 19.6363636364, expected 30": 1, "value is 53.803030303, expected 30": 1, "value is 768, expected 30": 1, "value is 249.5, expected 30": 1, "value is 2.83636363636, expected 30": 1, "value is 594, expected 30": 1, "value is 34.1666666667, expected 30": 1, "value is 78, expected 30": 1, "value is 57, expected 11": 1, "value is 11.6315789474, expected 24": 1, "value is 16, expected 24": 1, "value is 136, expected 24": 1, "value is 83.6, expected 24": 1, "value is 1012, expected 24": 1, "value is 162.962962963, expected 24": 1, "value is 2663.15789474, expected 24": 1, "value is 47.3469387755, expected 26": 1, "value is 2184, expected 26": 1, "value is 1395, expected 26": 1, "value is 56.9696969697, expected 26": 1, "value is -56.9696969697, expected 26": 1, "value is 1045, expected 26": 1, "value is 61, expected 35": 1, "value is 156, expected 35": 1, "value is 493, expected 35": 1, "value is 816, expected 35": 1, "value is 90, expected 35": 1, "value is 1496, expected 35": 1, "value is 88, expected 35": 1, "value is 1989, expected 35": 1, "value is 17, expected 59": 1, "value is 3612, expected 66": 1, "value is -2494, expected 66": 1, "value is 13.3661971831, expected 66": 1, "value is -154, expected 66": 1, "value is -2520, expected 66": 1, "value is 9.50704225352, expected 66": 1, "value is 2494, expected 66": 1, "value is -2, expected 15": 1, "value is 1534, expected 86": 1, "value is 989, expected 22": 1, "value is -67, expected 22": 1, "value is 1152, expected 22": 1, "value is 40.25, expected 22": 1, "value is 1012, expected 22": 1, "value is 38.8153846154, expected 83": 1, "value is 2458, expected 83": 1, "value is 3118, expected 83": 1, "value is 35.0597014925, expected 83": 1, "value is 38.7894736842, expected 83": 1, "value is 1436, expected 83": 1, "value is 3221.59701493, expected 83": 1, "value is -1072, expected 83": 1, "value is 276.705882353, expected 53": 1, "value is 2465, expected 53": 1, "value is 26.1071428571, expected 53": 1, "value is 358, expected 53": 1, "value is 9408, expected 53": 1, "value is -162.411764706, expected 53": 1, "value is 755, expected 53": 1, "value is 2568, expected 53": 1, "value is 3696, expected 53": 1, "value is 953, expected 53": 1, "value is 3900, expected 53": 1, "value is 912, expected 72": 1, "value is 164.25, expected 72": 1, "value is 282, expected 72": 1, "value is 611.375, expected 72": 1, "value is -1216, expected 78": 1, "value is 33.84375, expected 78": 1, "value is 359, expected 78": 1, "value is 535.783783784, expected 78": 1, "value is 103.529411765, expected 78": 1, "value is 857, expected 78": 1, "value is -63, expected 87": 1, "value is 10, expected 87": 1, "value is 153.797468354, expected 87": 1, "value is 229, expected 87": 1, "value is 94.9565217391, expected 58": 1, "value is 3059.2826087, expected 58": 1, "value is 959, expected 58": 1, "value is 6396, expected 58": 1, "value is 2423, expected 58": 1, "value is 1568, expected 58": 1, "value is 17.902173913, expected 58": 1, "value is -40.7213114754, expected 58": 1, "value is 4647.96078431, expected 58": 1, "value is 95.9090909091, expected 51": 1, "value is 158, expected 51": 1, "value is 452, expected 51": 1, "value is 12.253164557, expected 51": 1, "value is -160, expected 92": 1, "value is 15, expected 92": 1, "value is 7.125, expected 92": 1, "value is 96, expected 92": 1, "value is 11.5, expected 92": 1, "value is 98, expected 92": 1 }, "empty": false, "first_attempt_count": 100, "first_attempt_r1_format_rate": 1.0, "first_attempt_answer_tag_rate": 1.0, "first_attempt_allowed_chars_rate": 1.0, "first_attempt_parse_ok_rate": 0.98, "first_attempt_numbers_ok_rate": 0.51, "first_attempt_target_ok_rate": 0.02, "first_attempt_valid_rate": 0.01, "attempt_count": 1600, "attempt_r1_format_rate": 0.995625, "attempt_answer_tag_rate": 0.995625, "attempt_allowed_chars_rate": 0.98875, "attempt_parse_ok_rate": 0.9675, "attempt_numbers_ok_rate": 0.58375, "attempt_target_ok_rate": 0.02625, "attempt_valid_rate": 0.019375, "mean_candidates_until_solution": 8.055555555555555, "median_candidates_until_solution": 9.0, "max_candidates_until_solution": 15, "solved_at_1_count": 1, "solved_at_1": 0.01, "solved_at_4_count": 4, "solved_at_4": 0.04, "solved_at_16_count": 18, "solved_at_16": 0.18, "num_samples": 16 }