231 lines
8.3 KiB
JSON
231 lines
8.3 KiB
JSON
{
|
|
"count": 100,
|
|
"solved": 36,
|
|
"solved_rate": 0.36,
|
|
"solved_at_n_count": 36,
|
|
"solved_at_n_rate": 0.36,
|
|
"pass_at_1_count": 5,
|
|
"pass_at_1_rate": 0.05,
|
|
"candidate_count_total": 1600,
|
|
"candidate_count_mean_per_problem": 16,
|
|
"num_samples_max": 16,
|
|
"num_samples_min": 16,
|
|
"reason_counts": {
|
|
"ok": 69,
|
|
"value is 20, expected 24": 63,
|
|
"value is 48, expected 24": 61,
|
|
"value is 18, expected 24": 57,
|
|
"value is 40, expected 24": 53,
|
|
"value is 0, expected 24": 51,
|
|
"value is 6, expected 24": 49,
|
|
"value is 60, expected 24": 48,
|
|
"value is 16, expected 24": 45,
|
|
"value is 12, expected 24": 42,
|
|
"value is 30, expected 24": 41,
|
|
"value is 84, expected 24": 41,
|
|
"value is 8, expected 24": 38,
|
|
"value is 21, expected 24": 37,
|
|
"value is 22, expected 24": 37,
|
|
"value is 36, expected 24": 37,
|
|
"value is 72, expected 24": 34,
|
|
"numbers are not used exactly once": 31,
|
|
"value is 25, expected 24": 30,
|
|
"value is 26, expected 24": 28,
|
|
"value is 28, expected 24": 26,
|
|
"empty expression": 26,
|
|
"value is 42, expected 24": 25,
|
|
"value is 32, expected 24": 24,
|
|
"value is 14, expected 24": 24,
|
|
"value is 33, expected 24": 23,
|
|
"value is 27, expected 24": 22,
|
|
"value is 10, expected 24": 19,
|
|
"value is 23, expected 24": 16,
|
|
"value is 56, expected 24": 16,
|
|
"value is 19, expected 24": 14,
|
|
"value is 96, expected 24": 14,
|
|
"value is 66, expected 24": 13,
|
|
"value is 29, expected 24": 13,
|
|
"value is 52, expected 24": 12,
|
|
"value is 54, expected 24": 12,
|
|
"value is 9, expected 24": 12,
|
|
"value is 15, expected 24": 11,
|
|
"value is 35, expected 24": 11,
|
|
"value is 50, expected 24": 11,
|
|
"value is 120, expected 24": 11,
|
|
"value is 9.33333333333, expected 24": 10,
|
|
"value is 7, expected 24": 10,
|
|
"value is 31, expected 24": 10,
|
|
"value is 17, expected 24": 9,
|
|
"value is 44, expected 24": 9,
|
|
"value is 39, expected 24": 9,
|
|
"value is 37, expected 24": 8,
|
|
"value is 13, expected 24": 8,
|
|
"value is 10.5, expected 24": 8,
|
|
"value is 18.6666666667, expected 24": 7,
|
|
"value is 34, expected 24": 7,
|
|
"value is 90, expected 24": 7,
|
|
"value is 64, expected 24": 7,
|
|
"value is -12, expected 24": 7,
|
|
"value is 12.5, expected 24": 6,
|
|
"value is 43, expected 24": 6,
|
|
"value is 4, expected 24": 6,
|
|
"value is 70, expected 24": 6,
|
|
"value is 80, expected 24": 6,
|
|
"value is 104, expected 24": 5,
|
|
"value is 45, expected 24": 5,
|
|
"value is 22.5, expected 24": 4,
|
|
"value is 49, expected 24": 4,
|
|
"value is 20.5714285714, expected 24": 4,
|
|
"value is 5, expected 24": 4,
|
|
"value is 16.6666666667, expected 24": 4,
|
|
"value is 77, expected 24": 4,
|
|
"value is 91, expected 24": 4,
|
|
"value is 108, expected 24": 4,
|
|
"value is 25.6, expected 24": 4,
|
|
"value is 25.5, expected 24": 3,
|
|
"value is 46, expected 24": 3,
|
|
"value is -7, expected 24": 3,
|
|
"value is 12.8571428571, expected 24": 3,
|
|
"value is 65, expected 24": 3,
|
|
"value is 38, expected 24": 3,
|
|
"value is 16.5, expected 24": 3,
|
|
"value is 88, expected 24": 3,
|
|
"value is -8, expected 24": 3,
|
|
"value is 24.2, expected 24": 3,
|
|
"value is -16, expected 24": 3,
|
|
"value is 33.3333333333, expected 24": 3,
|
|
"value is 26.4, expected 24": 3,
|
|
"value is 22.4, expected 24": 2,
|
|
"value is 63, expected 24": 2,
|
|
"value is 105, expected 24": 2,
|
|
"value is 4.36363636364, expected 24": 2,
|
|
"value is 51.0769230769, expected 24": 2,
|
|
"value is 10.8, expected 24": 2,
|
|
"value is 6.4, expected 24": 2,
|
|
"value is 6.5, expected 24": 2,
|
|
"value is 25.6666666667, expected 24": 2,
|
|
"value is 50.6666666667, expected 24": 2,
|
|
"value is 8.33333333333, expected 24": 2,
|
|
"contains disallowed characters": 2,
|
|
"value is -36, expected 24": 2,
|
|
"value is 17.7777777778, expected 24": 2,
|
|
"value is 7.33333333333, expected 24": 2,
|
|
"value is 69.3333333333, expected 24": 2,
|
|
"value is 132, expected 24": 2,
|
|
"value is 14.6666666667, expected 24": 2,
|
|
"value is -6, expected 24": 1,
|
|
"value is -26, expected 24": 1,
|
|
"value is 7.61538461538, expected 24": 1,
|
|
"value is 23.1111111111, expected 24": 1,
|
|
"value is 117, expected 24": 1,
|
|
"value is 24.5, expected 24": 1,
|
|
"value is 22.2857142857, expected 24": 1,
|
|
"value is 9.42857142857, expected 24": 1,
|
|
"value is 78, expected 24": 1,
|
|
"value is 51.3333333333, expected 24": 1,
|
|
"value is 5.45454545455, expected 24": 1,
|
|
"value is 8.4, expected 24": 1,
|
|
"value is 7.14285714286, expected 24": 1,
|
|
"value is 27.1428571429, expected 24": 1,
|
|
"value is 5.6, expected 24": 1,
|
|
"value is 98, expected 24": 1,
|
|
"value is 23.7777777778, expected 24": 1,
|
|
"value is 115.909090909, expected 24": 1,
|
|
"value is 29.3333333333, expected 24": 1,
|
|
"value is 240, expected 24": 1,
|
|
"value is 136, expected 24": 1,
|
|
"value is 19.2, expected 24": 1,
|
|
"value is 5.71428571429, expected 24": 1,
|
|
"value is 22.1428571429, expected 24": 1,
|
|
"value is 95, expected 24": 1,
|
|
"value is 23.5, expected 24": 1,
|
|
"value is 8.125, expected 24": 1,
|
|
"value is 160, expected 24": 1,
|
|
"value is 23.6, expected 24": 1,
|
|
"value is 27.4, expected 24": 1,
|
|
"value is 35.2, expected 24": 1,
|
|
"value is 53.5714285714, expected 24": 1,
|
|
"value is 22.2, expected 24": 1,
|
|
"value is 13.5, expected 24": 1,
|
|
"value is 28.1666666667, expected 24": 1,
|
|
"value is -136, expected 24": 1,
|
|
"value is 11.2, expected 24": 1,
|
|
"value is 24.6153846154, expected 24": 1,
|
|
"value is 21.5, expected 24": 1,
|
|
"value is 11, expected 24": 1,
|
|
"value is 74.7692307692, expected 24": 1,
|
|
"value is 4.91666666667, expected 24": 1,
|
|
"value is 47, expected 24": 1,
|
|
"value is 65.5, expected 24": 1,
|
|
"value is 11.5384615385, expected 24": 1,
|
|
"value is 11.25, expected 24": 1,
|
|
"value is 21.6363636364, expected 24": 1,
|
|
"value is 165, expected 24": 1,
|
|
"value is 220, expected 24": 1,
|
|
"value is 33.5, expected 24": 1,
|
|
"value is 8.66666666667, expected 24": 1,
|
|
"value is 3, expected 24": 1,
|
|
"value is 23.3333333333, expected 24": 1,
|
|
"value is 55, expected 24": 1,
|
|
"value is 6.46153846154, expected 24": 1,
|
|
"value is 21.3333333333, expected 24": 1,
|
|
"value is 168, expected 24": 1,
|
|
"value is 32.5714285714, expected 24": 1,
|
|
"value is 11.6666666667, expected 24": 1,
|
|
"value is 40.4285714286, expected 24": 1,
|
|
"value is 93, expected 24": 1,
|
|
"value is 24.4, expected 24": 1,
|
|
"value is 8.55555555556, expected 24": 1,
|
|
"value is 57, expected 24": 1,
|
|
"value is -48, expected 24": 1,
|
|
"value is 9.6, expected 24": 1,
|
|
"value is 18.8571428571, expected 24": 1,
|
|
"value is 89.1428571429, expected 24": 1,
|
|
"value is 144, expected 24": 1,
|
|
"value is 52.25, expected 24": 1,
|
|
"value is 16.2, expected 24": 1,
|
|
"value is 38.6666666667, expected 24": 1,
|
|
"value is -9, expected 24": 1,
|
|
"value is 4.66666666667, expected 24": 1,
|
|
"value is 126, expected 24": 1,
|
|
"value is 12.7272727273, expected 24": 1,
|
|
"value is 22.25, expected 24": 1,
|
|
"value is 85.3333333333, expected 24": 1,
|
|
"value is 4.58333333333, expected 24": 1,
|
|
"value is -56, expected 24": 1,
|
|
"division by zero": 1,
|
|
"value is 33.2, expected 24": 1,
|
|
"value is 26.5, expected 24": 1,
|
|
"value is -91, expected 24": 1,
|
|
"value is 25.3333333333, expected 24": 1,
|
|
"value is 75, expected 24": 1
|
|
},
|
|
"empty": false,
|
|
"first_attempt_count": 100,
|
|
"first_attempt_r1_format_rate": 0.98,
|
|
"first_attempt_answer_tag_rate": 0.98,
|
|
"first_attempt_allowed_chars_rate": 0.97,
|
|
"first_attempt_parse_ok_rate": 0.97,
|
|
"first_attempt_numbers_ok_rate": 0.96,
|
|
"first_attempt_target_ok_rate": 0.05,
|
|
"first_attempt_valid_rate": 0.05,
|
|
"attempt_count": 1600,
|
|
"attempt_r1_format_rate": 0.98375,
|
|
"attempt_answer_tag_rate": 0.98375,
|
|
"attempt_allowed_chars_rate": 0.9825,
|
|
"attempt_parse_ok_rate": 0.981875,
|
|
"attempt_numbers_ok_rate": 0.96375,
|
|
"attempt_target_ok_rate": 0.045625,
|
|
"attempt_valid_rate": 0.043125,
|
|
"mean_candidates_until_solution": 5.888888888888889,
|
|
"median_candidates_until_solution": 3.0,
|
|
"max_candidates_until_solution": 16,
|
|
"solved_at_1_count": 5,
|
|
"solved_at_1": 0.05,
|
|
"solved_at_4_count": 20,
|
|
"solved_at_4": 0.2,
|
|
"solved_at_16_count": 36,
|
|
"solved_at_16": 0.36,
|
|
"num_samples": 16
|
|
}
|