61 lines
1.9 KiB
JSON
61 lines
1.9 KiB
JSON
{
|
|
"count": 100,
|
|
"solved": 0,
|
|
"solved_rate": 0.0,
|
|
"solved_at_n_count": 0,
|
|
"solved_at_n_rate": 0.0,
|
|
"pass_at_1_count": 0,
|
|
"pass_at_1_rate": 0.0,
|
|
"candidate_count_total": 100,
|
|
"candidate_count_mean_per_problem": 1,
|
|
"num_samples_max": 1,
|
|
"num_samples_min": 1,
|
|
"reason_counts": {
|
|
"empty expression": 43,
|
|
"contains disallowed characters": 20,
|
|
"numbers are not used exactly once": 17,
|
|
"value is 34, expected 24": 2,
|
|
"value is 16, expected 24": 1,
|
|
"value is 40, expected 24": 1,
|
|
"value is 6, expected 24": 1,
|
|
"value is 70, expected 24": 1,
|
|
"value is 28, expected 24": 1,
|
|
"unsupported expression node: UnaryOp": 1,
|
|
"value is 5, expected 24": 1,
|
|
"value is -0.875, expected 24": 1,
|
|
"value is 25, expected 24": 1,
|
|
"value is 32, expected 24": 1,
|
|
"value is 153, expected 24": 1,
|
|
"value is 6.85714285714, expected 24": 1,
|
|
"value is 89, expected 24": 1,
|
|
"value is 96, expected 24": 1,
|
|
"value is 23, expected 24": 1,
|
|
"value is 58.5, expected 24": 1,
|
|
"value is 30, expected 24": 1,
|
|
"value is 26, expected 24": 1
|
|
},
|
|
"empty": false,
|
|
"first_attempt_count": 100,
|
|
"first_attempt_r1_format_rate": 0.65,
|
|
"first_attempt_answer_tag_rate": 0.65,
|
|
"first_attempt_allowed_chars_rate": 0.37,
|
|
"first_attempt_parse_ok_rate": 0.36,
|
|
"first_attempt_numbers_ok_rate": 0.19,
|
|
"first_attempt_target_ok_rate": 0.03,
|
|
"first_attempt_valid_rate": 0.0,
|
|
"attempt_count": 100,
|
|
"attempt_r1_format_rate": 0.65,
|
|
"attempt_answer_tag_rate": 0.65,
|
|
"attempt_allowed_chars_rate": 0.37,
|
|
"attempt_parse_ok_rate": 0.36,
|
|
"attempt_numbers_ok_rate": 0.19,
|
|
"attempt_target_ok_rate": 0.03,
|
|
"attempt_valid_rate": 0.0,
|
|
"mean_candidates_until_solution": null,
|
|
"median_candidates_until_solution": null,
|
|
"max_candidates_until_solution": null,
|
|
"solved_at_1_count": 0,
|
|
"solved_at_1": 0.0,
|
|
"num_samples": 1
|
|
}
|