{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 2.2222222222222223, "eval_steps": 500, "global_step": 20, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 63.0, "completions/max_terminated_length": 63.0, "completions/mean_length": 40.75, "completions/mean_terminated_length": 40.75, "completions/min_length": 31.0, "completions/min_terminated_length": 31.0, "epoch": 0.1111111111111111, "grad_norm": 25.69437599182129, "kl": 0.0, "learning_rate": 0.0, "loss": -0.1171, "num_tokens": 675.0, "reward": 0.5159521698951721, "reward_std": 0.0688270628452301, "rewards/concensus_correctness_reward_func/mean": 0.0, "rewards/concensus_correctness_reward_func/std": 0.0, "rewards/consensus_reward_func/mean": 0.0, "rewards/consensus_reward_func/std": 0.0, "rewards/cumulative_reward_2/mean": 0.0, "rewards/cumulative_reward_2/std": 0.0, "rewards/final_correctness_reward_func/mean": 0.0, "rewards/final_correctness_reward_func/std": 0.0, "rewards/question_recreation_reward_func/mean": 0.07845214009284973, "rewards/question_recreation_reward_func/std": 0.04025953263044357, "rewards/soft_format_reward_func/mean": 0.0, "rewards/soft_format_reward_func/std": 0.0, "rewards/strict_format_reward_func/mean": 0.0, "rewards/strict_format_reward_func/std": 0.0, "rewards/xmlcount_reward_func/mean": 0.4375, "rewards/xmlcount_reward_func/std": 0.125, "step": 1 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 104.0, "completions/max_terminated_length": 104.0, "completions/mean_length": 56.5, "completions/mean_terminated_length": 56.5, "completions/min_length": 24.0, "completions/min_terminated_length": 24.0, "epoch": 0.2222222222222222, "grad_norm": 22.427824020385742, "kl": 0.0, "learning_rate": 1e-06, "loss": 0.0564, "num_tokens": 1413.0, "reward": 0.5325700640678406, "reward_std": 0.0006790067418478429, "rewards/concensus_correctness_reward_func/mean": 0.0, "rewards/concensus_correctness_reward_func/std": 0.0, "rewards/consensus_reward_func/mean": 0.0, "rewards/consensus_reward_func/std": 0.0, "rewards/cumulative_reward_2/mean": 0.0, "rewards/cumulative_reward_2/std": 0.0, "rewards/final_correctness_reward_func/mean": 0.0, "rewards/final_correctness_reward_func/std": 0.0, "rewards/question_recreation_reward_func/mean": 0.032570019364356995, "rewards/question_recreation_reward_func/std": 0.0007998892688192427, "rewards/soft_format_reward_func/mean": 0.0, "rewards/soft_format_reward_func/std": 0.0, "rewards/strict_format_reward_func/mean": 0.0, "rewards/strict_format_reward_func/std": 0.0, "rewards/xmlcount_reward_func/mean": 0.5, "rewards/xmlcount_reward_func/std": 0.0, "step": 2 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 231.0, "completions/max_terminated_length": 231.0, "completions/mean_length": 129.5, "completions/mean_terminated_length": 129.5, "completions/min_length": 59.0, "completions/min_terminated_length": 59.0, "epoch": 0.3333333333333333, "grad_norm": 18.40679168701172, "kl": 0.002804491203278303, "learning_rate": 9.931806517013612e-07, "loss": 0.297, "num_tokens": 2443.0, "reward": 0.6747503876686096, "reward_std": 0.1533203423023224, "rewards/concensus_correctness_reward_func/mean": 0.0, "rewards/concensus_correctness_reward_func/std": 0.0, "rewards/consensus_reward_func/mean": 0.0, "rewards/consensus_reward_func/std": 0.0, "rewards/cumulative_reward_2/mean": 0.0, "rewards/cumulative_reward_2/std": 0.0, "rewards/final_correctness_reward_func/mean": 0.0, "rewards/final_correctness_reward_func/std": 0.0, "rewards/question_recreation_reward_func/mean": 0.20650038123130798, "rewards/question_recreation_reward_func/std": 0.019515778869390488, "rewards/soft_format_reward_func/mean": 0.0, "rewards/soft_format_reward_func/std": 0.0, "rewards/strict_format_reward_func/mean": 0.0, "rewards/strict_format_reward_func/std": 0.0, "rewards/xmlcount_reward_func/mean": 0.46825000643730164, "rewards/xmlcount_reward_func/std": 0.21338286995887756, "step": 3 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 110.0, "completions/max_terminated_length": 110.0, "completions/mean_length": 62.25, "completions/mean_terminated_length": 62.25, "completions/min_length": 23.0, "completions/min_terminated_length": 23.0, "epoch": 0.4444444444444444, "grad_norm": 22.179134368896484, "kl": 0.012003686744719744, "learning_rate": 9.729086208503173e-07, "loss": -0.0112, "num_tokens": 3204.0, "reward": 0.6682786345481873, "reward_std": 0.016284530982375145, "rewards/concensus_correctness_reward_func/mean": 0.0, "rewards/concensus_correctness_reward_func/std": 0.0, "rewards/consensus_reward_func/mean": 0.0, "rewards/consensus_reward_func/std": 0.0, "rewards/cumulative_reward_2/mean": 0.0, "rewards/cumulative_reward_2/std": 0.0, "rewards/final_correctness_reward_func/mean": 0.0, "rewards/final_correctness_reward_func/std": 0.0, "rewards/question_recreation_reward_func/mean": 0.04327860474586487, "rewards/question_recreation_reward_func/std": 0.0372861810028553, "rewards/soft_format_reward_func/mean": 0.0, "rewards/soft_format_reward_func/std": 0.0, "rewards/strict_format_reward_func/mean": 0.0, "rewards/strict_format_reward_func/std": 0.0, "rewards/xmlcount_reward_func/mean": 0.625, "rewards/xmlcount_reward_func/std": 0.14433756470680237, "step": 4 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 79.0, "completions/max_terminated_length": 79.0, "completions/mean_length": 54.75, "completions/mean_terminated_length": 54.75, "completions/min_length": 26.0, "completions/min_terminated_length": 26.0, "epoch": 0.5555555555555556, "grad_norm": 19.958255767822266, "kl": 0.011007877299562097, "learning_rate": 9.397368756032444e-07, "loss": -0.1713, "num_tokens": 3935.0, "reward": 0.8996135592460632, "reward_std": 0.049514204263687134, "rewards/concensus_correctness_reward_func/mean": 0.0, "rewards/concensus_correctness_reward_func/std": 0.0, "rewards/consensus_reward_func/mean": 0.0, "rewards/consensus_reward_func/std": 0.0, "rewards/cumulative_reward_2/mean": 0.0, "rewards/cumulative_reward_2/std": 0.0, "rewards/final_correctness_reward_func/mean": 0.0, "rewards/final_correctness_reward_func/std": 0.0, "rewards/question_recreation_reward_func/mean": 0.27461355924606323, "rewards/question_recreation_reward_func/std": 0.044617120176553726, "rewards/soft_format_reward_func/mean": 0.0, "rewards/soft_format_reward_func/std": 0.0, "rewards/strict_format_reward_func/mean": 0.0, "rewards/strict_format_reward_func/std": 0.0, "rewards/xmlcount_reward_func/mean": 0.625, "rewards/xmlcount_reward_func/std": 0.14433756470680237, "step": 5 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 29.0, "completions/max_terminated_length": 29.0, "completions/mean_length": 27.25, "completions/mean_terminated_length": 27.25, "completions/min_length": 24.0, "completions/min_terminated_length": 24.0, "epoch": 0.6666666666666666, "grad_norm": 23.830913543701172, "kl": 0.04720709100365639, "learning_rate": 8.945702546981968e-07, "loss": -0.0311, "num_tokens": 4556.0, "reward": 0.5360786318778992, "reward_std": 0.008525591343641281, "rewards/concensus_correctness_reward_func/mean": 0.0, "rewards/concensus_correctness_reward_func/std": 0.0, "rewards/consensus_reward_func/mean": 0.0, "rewards/consensus_reward_func/std": 0.0, "rewards/cumulative_reward_2/mean": 0.0, "rewards/cumulative_reward_2/std": 0.0, "rewards/final_correctness_reward_func/mean": 0.0, "rewards/final_correctness_reward_func/std": 0.0, "rewards/question_recreation_reward_func/mean": 0.03607860952615738, "rewards/question_recreation_reward_func/std": 0.02430449239909649, "rewards/soft_format_reward_func/mean": 0.0, "rewards/soft_format_reward_func/std": 0.0, "rewards/strict_format_reward_func/mean": 0.0, "rewards/strict_format_reward_func/std": 0.0, "rewards/xmlcount_reward_func/mean": 0.5, "rewards/xmlcount_reward_func/std": 0.0, "step": 6 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 56.0, "completions/max_terminated_length": 56.0, "completions/mean_length": 40.5, "completions/mean_terminated_length": 40.5, "completions/min_length": 30.0, "completions/min_terminated_length": 30.0, "epoch": 0.7777777777777778, "grad_norm": 39.75885772705078, "kl": 0.056546732783317566, "learning_rate": 8.386407858128706e-07, "loss": -0.07, "num_tokens": 5230.0, "reward": 0.6005383729934692, "reward_std": 0.08559251576662064, "rewards/concensus_correctness_reward_func/mean": 0.0, "rewards/concensus_correctness_reward_func/std": 0.0, "rewards/consensus_reward_func/mean": 0.0, "rewards/consensus_reward_func/std": 0.0, "rewards/cumulative_reward_2/mean": 0.0, "rewards/cumulative_reward_2/std": 0.0, "rewards/final_correctness_reward_func/mean": 0.0, "rewards/final_correctness_reward_func/std": 0.0, "rewards/question_recreation_reward_func/mean": 0.03853834420442581, "rewards/question_recreation_reward_func/std": 0.007710414472967386, "rewards/soft_format_reward_func/mean": 0.0, "rewards/soft_format_reward_func/std": 0.0, "rewards/strict_format_reward_func/mean": 0.0, "rewards/strict_format_reward_func/std": 0.0, "rewards/xmlcount_reward_func/mean": 0.562000036239624, "rewards/xmlcount_reward_func/std": 0.12400001287460327, "step": 7 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 95.0, "completions/max_terminated_length": 95.0, "completions/mean_length": 61.25, "completions/mean_terminated_length": 61.25, "completions/min_length": 30.0, "completions/min_terminated_length": 30.0, "epoch": 0.8888888888888888, "grad_norm": 25.805145263671875, "kl": 0.07301068678498268, "learning_rate": 7.734740790612136e-07, "loss": -0.3117, "num_tokens": 5987.0, "reward": 0.7229201197624207, "reward_std": 0.0429324246942997, "rewards/concensus_correctness_reward_func/mean": 0.0, "rewards/concensus_correctness_reward_func/std": 0.0, "rewards/consensus_reward_func/mean": 0.0, "rewards/consensus_reward_func/std": 0.0, "rewards/cumulative_reward_2/mean": 0.0, "rewards/cumulative_reward_2/std": 0.0, "rewards/final_correctness_reward_func/mean": 0.0, "rewards/final_correctness_reward_func/std": 0.0, "rewards/question_recreation_reward_func/mean": 0.22292014956474304, "rewards/question_recreation_reward_func/std": 0.05922666937112808, "rewards/soft_format_reward_func/mean": 0.0, "rewards/soft_format_reward_func/std": 0.0, "rewards/strict_format_reward_func/mean": 0.0, "rewards/strict_format_reward_func/std": 0.0, "rewards/xmlcount_reward_func/mean": 0.5, "rewards/xmlcount_reward_func/std": 0.0, "step": 8 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 47.0, "completions/max_terminated_length": 47.0, "completions/mean_length": 35.5, "completions/mean_terminated_length": 35.5, "completions/min_length": 24.0, "completions/min_terminated_length": 24.0, "epoch": 1.0, "grad_norm": 30.85164451599121, "kl": 0.02306750975549221, "learning_rate": 7.008477123264847e-07, "loss": 0.0617, "num_tokens": 6634.0, "reward": 0.5571061372756958, "reward_std": 0.00527472048997879, "rewards/concensus_correctness_reward_func/mean": 0.0, "rewards/concensus_correctness_reward_func/std": 0.0, "rewards/consensus_reward_func/mean": 0.0, "rewards/consensus_reward_func/std": 0.0, "rewards/cumulative_reward_2/mean": 0.0, "rewards/cumulative_reward_2/std": 0.0, "rewards/final_correctness_reward_func/mean": 0.0, "rewards/final_correctness_reward_func/std": 0.0, "rewards/question_recreation_reward_func/mean": 0.0571061372756958, "rewards/question_recreation_reward_func/std": 0.011652503162622452, "rewards/soft_format_reward_func/mean": 0.0, "rewards/soft_format_reward_func/std": 0.0, "rewards/strict_format_reward_func/mean": 0.0, "rewards/strict_format_reward_func/std": 0.0, "rewards/xmlcount_reward_func/mean": 0.5, "rewards/xmlcount_reward_func/std": 0.0, "step": 9 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 99.0, "completions/max_terminated_length": 99.0, "completions/mean_length": 59.5, "completions/mean_terminated_length": 59.5, "completions/min_length": 31.0, "completions/min_terminated_length": 31.0, "epoch": 1.1111111111111112, "grad_norm": 18.589075088500977, "kl": 0.057917265221476555, "learning_rate": 6.227427435703995e-07, "loss": -0.2448, "num_tokens": 7384.0, "reward": 0.7360702753067017, "reward_std": 0.04403897002339363, "rewards/concensus_correctness_reward_func/mean": 0.0, "rewards/concensus_correctness_reward_func/std": 0.0, "rewards/consensus_reward_func/mean": 0.0, "rewards/consensus_reward_func/std": 0.0, "rewards/cumulative_reward_2/mean": 0.0, "rewards/cumulative_reward_2/std": 0.0, "rewards/final_correctness_reward_func/mean": 0.0, "rewards/final_correctness_reward_func/std": 0.0, "rewards/question_recreation_reward_func/mean": 0.23607021570205688, "rewards/question_recreation_reward_func/std": 0.07128877192735672, "rewards/soft_format_reward_func/mean": 0.0, "rewards/soft_format_reward_func/std": 0.0, "rewards/strict_format_reward_func/mean": 0.0, "rewards/strict_format_reward_func/std": 0.0, "rewards/xmlcount_reward_func/mean": 0.5, "rewards/xmlcount_reward_func/std": 0.0, "step": 10 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 31.0, "completions/max_terminated_length": 31.0, "completions/mean_length": 28.0, "completions/mean_terminated_length": 28.0, "completions/min_length": 25.0, "completions/min_terminated_length": 25.0, "epoch": 1.2222222222222223, "grad_norm": 34.077091217041016, "kl": 0.13308210298419, "learning_rate": 5.412896727361662e-07, "loss": 0.0104, "num_tokens": 8008.0, "reward": 0.53204345703125, "reward_std": 0.0063663022592663765, "rewards/concensus_correctness_reward_func/mean": 0.0, "rewards/concensus_correctness_reward_func/std": 0.0, "rewards/consensus_reward_func/mean": 0.0, "rewards/consensus_reward_func/std": 0.0, "rewards/cumulative_reward_2/mean": 0.0, "rewards/cumulative_reward_2/std": 0.0, "rewards/final_correctness_reward_func/mean": 0.0, "rewards/final_correctness_reward_func/std": 0.0, "rewards/question_recreation_reward_func/mean": 0.032043468207120895, "rewards/question_recreation_reward_func/std": 0.016084665432572365, "rewards/soft_format_reward_func/mean": 0.0, "rewards/soft_format_reward_func/std": 0.0, "rewards/strict_format_reward_func/mean": 0.0, "rewards/strict_format_reward_func/std": 0.0, "rewards/xmlcount_reward_func/mean": 0.5, "rewards/xmlcount_reward_func/std": 0.0, "step": 11 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 69.0, "completions/max_terminated_length": 69.0, "completions/mean_length": 39.5, "completions/mean_terminated_length": 39.5, "completions/min_length": 22.0, "completions/min_terminated_length": 22.0, "epoch": 1.3333333333333333, "grad_norm": 19.56139373779297, "kl": 0.09075122326612473, "learning_rate": 4.5871032726383385e-07, "loss": -0.2792, "num_tokens": 8678.0, "reward": 0.5326830744743347, "reward_std": 0.011520188301801682, "rewards/concensus_correctness_reward_func/mean": 0.0, "rewards/concensus_correctness_reward_func/std": 0.0, "rewards/consensus_reward_func/mean": 0.0, "rewards/consensus_reward_func/std": 0.0, "rewards/cumulative_reward_2/mean": 0.0, "rewards/cumulative_reward_2/std": 0.0, "rewards/final_correctness_reward_func/mean": 0.0, "rewards/final_correctness_reward_func/std": 0.0, "rewards/question_recreation_reward_func/mean": 0.032683055847883224, "rewards/question_recreation_reward_func/std": 0.009901088662445545, "rewards/soft_format_reward_func/mean": 0.0, "rewards/soft_format_reward_func/std": 0.0, "rewards/strict_format_reward_func/mean": 0.0, "rewards/strict_format_reward_func/std": 0.0, "rewards/xmlcount_reward_func/mean": 0.5, "rewards/xmlcount_reward_func/std": 0.0, "step": 12 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 89.0, "completions/max_terminated_length": 89.0, "completions/mean_length": 79.5, "completions/mean_terminated_length": 79.5, "completions/min_length": 70.0, "completions/min_terminated_length": 70.0, "epoch": 1.4444444444444444, "grad_norm": 26.145700454711914, "kl": 0.027578551787883043, "learning_rate": 3.772572564296004e-07, "loss": -0.0213, "num_tokens": 9508.0, "reward": 0.8646507263183594, "reward_std": 0.08279794454574585, "rewards/concensus_correctness_reward_func/mean": 0.0, "rewards/concensus_correctness_reward_func/std": 0.0, "rewards/consensus_reward_func/mean": 0.0, "rewards/consensus_reward_func/std": 0.0, "rewards/cumulative_reward_2/mean": 0.0, "rewards/cumulative_reward_2/std": 0.0, "rewards/final_correctness_reward_func/mean": 0.0, "rewards/final_correctness_reward_func/std": 0.0, "rewards/question_recreation_reward_func/mean": 0.1771506667137146, "rewards/question_recreation_reward_func/std": 0.056235793977975845, "rewards/soft_format_reward_func/mean": 0.0, "rewards/soft_format_reward_func/std": 0.0, "rewards/strict_format_reward_func/mean": 0.0, "rewards/strict_format_reward_func/std": 0.0, "rewards/xmlcount_reward_func/mean": 0.6875, "rewards/xmlcount_reward_func/std": 0.125, "step": 13 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 127.0, "completions/max_terminated_length": 127.0, "completions/mean_length": 92.75, "completions/mean_terminated_length": 92.75, "completions/min_length": 68.0, "completions/min_terminated_length": 68.0, "epoch": 1.5555555555555556, "grad_norm": 19.497051239013672, "kl": 0.03215572563931346, "learning_rate": 2.9915228767351535e-07, "loss": -0.084, "num_tokens": 10391.0, "reward": 0.6083483099937439, "reward_std": 0.10050921142101288, "rewards/concensus_correctness_reward_func/mean": 0.0, "rewards/concensus_correctness_reward_func/std": 0.0, "rewards/consensus_reward_func/mean": 0.0, "rewards/consensus_reward_func/std": 0.0, "rewards/cumulative_reward_2/mean": 0.0, "rewards/cumulative_reward_2/std": 0.0, "rewards/final_correctness_reward_func/mean": 0.0, "rewards/final_correctness_reward_func/std": 0.0, "rewards/question_recreation_reward_func/mean": 0.04584832862019539, "rewards/question_recreation_reward_func/std": 0.010448978282511234, "rewards/soft_format_reward_func/mean": 0.0, "rewards/soft_format_reward_func/std": 0.0, "rewards/strict_format_reward_func/mean": 0.0, "rewards/strict_format_reward_func/std": 0.0, "rewards/xmlcount_reward_func/mean": 0.5625, "rewards/xmlcount_reward_func/std": 0.125, "step": 14 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.25, "completions/max_length": 448.0, "completions/max_terminated_length": 47.0, "completions/mean_length": 138.75, "completions/mean_terminated_length": 35.66666793823242, "completions/min_length": 30.0, "completions/min_terminated_length": 30.0, "epoch": 1.6666666666666665, "grad_norm": 14.958171844482422, "kl": 0.036621191538870335, "learning_rate": 2.2652592093878665e-07, "loss": 0.2878, "num_tokens": 11458.0, "reward": 0.45188108086586, "reward_std": 0.15637479722499847, "rewards/concensus_correctness_reward_func/mean": 0.0, "rewards/concensus_correctness_reward_func/std": 0.0, "rewards/consensus_reward_func/mean": 0.0, "rewards/consensus_reward_func/std": 0.0, "rewards/cumulative_reward_2/mean": 0.0, "rewards/cumulative_reward_2/std": 0.0, "rewards/final_correctness_reward_func/mean": 0.0, "rewards/final_correctness_reward_func/std": 0.0, "rewards/question_recreation_reward_func/mean": 0.04563111066818237, "rewards/question_recreation_reward_func/std": 0.020204132422804832, "rewards/soft_format_reward_func/mean": 0.0, "rewards/soft_format_reward_func/std": 0.0, "rewards/strict_format_reward_func/mean": 0.0, "rewards/strict_format_reward_func/std": 0.0, "rewards/xmlcount_reward_func/mean": 0.40625, "rewards/xmlcount_reward_func/std": 0.1875, "step": 15 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 55.0, "completions/max_terminated_length": 55.0, "completions/mean_length": 37.25, "completions/mean_terminated_length": 37.25, "completions/min_length": 29.0, "completions/min_terminated_length": 29.0, "epoch": 1.7777777777777777, "grad_norm": 27.896305084228516, "kl": 0.17185895144939423, "learning_rate": 1.6135921418712955e-07, "loss": -0.064, "num_tokens": 12119.0, "reward": 0.4707985520362854, "reward_std": 0.27111726999282837, "rewards/concensus_correctness_reward_func/mean": 0.0, "rewards/concensus_correctness_reward_func/std": 0.0, "rewards/consensus_reward_func/mean": 0.0, "rewards/consensus_reward_func/std": 0.0, "rewards/cumulative_reward_2/mean": 0.0, "rewards/cumulative_reward_2/std": 0.0, "rewards/final_correctness_reward_func/mean": 0.0, "rewards/final_correctness_reward_func/std": 0.0, "rewards/question_recreation_reward_func/mean": 0.03329858183860779, "rewards/question_recreation_reward_func/std": 0.012222301214933395, "rewards/soft_format_reward_func/mean": 0.0, "rewards/soft_format_reward_func/std": 0.0, "rewards/strict_format_reward_func/mean": 0.0, "rewards/strict_format_reward_func/std": 0.0, "rewards/xmlcount_reward_func/mean": 0.4375, "rewards/xmlcount_reward_func/std": 0.23935678601264954, "step": 16 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 96.0, "completions/max_terminated_length": 96.0, "completions/mean_length": 68.5, "completions/mean_terminated_length": 68.5, "completions/min_length": 27.0, "completions/min_terminated_length": 27.0, "epoch": 1.8888888888888888, "grad_norm": 17.811830520629883, "kl": 0.056419532746076584, "learning_rate": 1.0542974530180327e-07, "loss": -0.2264, "num_tokens": 12905.0, "reward": 1.2158468961715698, "reward_std": 0.8011028170585632, "rewards/concensus_correctness_reward_func/mean": 0.0, "rewards/concensus_correctness_reward_func/std": 0.0, "rewards/consensus_reward_func/mean": 0.0, "rewards/consensus_reward_func/std": 0.0, "rewards/cumulative_reward_2/mean": 0.0, "rewards/cumulative_reward_2/std": 0.0, "rewards/final_correctness_reward_func/mean": 0.5, "rewards/final_correctness_reward_func/std": 1.0, "rewards/question_recreation_reward_func/mean": 0.02884695678949356, "rewards/question_recreation_reward_func/std": 0.008567274548113346, "rewards/soft_format_reward_func/mean": 0.0, "rewards/soft_format_reward_func/std": 0.0, "rewards/strict_format_reward_func/mean": 0.0, "rewards/strict_format_reward_func/std": 0.0, "rewards/xmlcount_reward_func/mean": 0.687000036239624, "rewards/xmlcount_reward_func/std": 0.12467022985219955, "step": 17 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 59.0, "completions/max_terminated_length": 59.0, "completions/mean_length": 56.5, "completions/mean_terminated_length": 56.5, "completions/min_length": 54.0, "completions/min_terminated_length": 54.0, "epoch": 2.0, "grad_norm": 24.79212188720703, "kl": 0.0544373020529747, "learning_rate": 6.026312439675551e-08, "loss": 0.0433, "num_tokens": 13653.0, "reward": 0.6995986700057983, "reward_std": 0.012008922174572945, "rewards/concensus_correctness_reward_func/mean": 0.0, "rewards/concensus_correctness_reward_func/std": 0.0, "rewards/consensus_reward_func/mean": 0.0, "rewards/consensus_reward_func/std": 0.0, "rewards/cumulative_reward_2/mean": 0.0, "rewards/cumulative_reward_2/std": 0.0, "rewards/final_correctness_reward_func/mean": 0.0, "rewards/final_correctness_reward_func/std": 0.0, "rewards/question_recreation_reward_func/mean": 0.07459867000579834, "rewards/question_recreation_reward_func/std": 0.05076611042022705, "rewards/soft_format_reward_func/mean": 0.0, "rewards/soft_format_reward_func/std": 0.0, "rewards/strict_format_reward_func/mean": 0.0, "rewards/strict_format_reward_func/std": 0.0, "rewards/xmlcount_reward_func/mean": 0.625, "rewards/xmlcount_reward_func/std": 0.14433756470680237, "step": 18 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 79.0, "completions/max_terminated_length": 79.0, "completions/mean_length": 65.5, "completions/mean_terminated_length": 65.5, "completions/min_length": 55.0, "completions/min_terminated_length": 55.0, "epoch": 2.111111111111111, "grad_norm": 23.983592987060547, "kl": 0.030249727424234152, "learning_rate": 2.7091379149682682e-08, "loss": 0.07, "num_tokens": 14427.0, "reward": 0.9142850637435913, "reward_std": 0.027192028239369392, "rewards/concensus_correctness_reward_func/mean": 0.0, "rewards/concensus_correctness_reward_func/std": 0.0, "rewards/consensus_reward_func/mean": 0.0, "rewards/consensus_reward_func/std": 0.0, "rewards/cumulative_reward_2/mean": 0.0, "rewards/cumulative_reward_2/std": 0.0, "rewards/final_correctness_reward_func/mean": 0.0, "rewards/final_correctness_reward_func/std": 0.0, "rewards/question_recreation_reward_func/mean": 0.2892850637435913, "rewards/question_recreation_reward_func/std": 0.027173683047294617, "rewards/soft_format_reward_func/mean": 0.0, "rewards/soft_format_reward_func/std": 0.0, "rewards/strict_format_reward_func/mean": 0.0, "rewards/strict_format_reward_func/std": 0.0, "rewards/xmlcount_reward_func/mean": 0.625, "rewards/xmlcount_reward_func/std": 0.14433756470680237, "step": 19 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 85.0, "completions/max_terminated_length": 85.0, "completions/mean_length": 56.25, "completions/mean_terminated_length": 56.25, "completions/min_length": 33.0, "completions/min_terminated_length": 33.0, "epoch": 2.2222222222222223, "grad_norm": 19.803834915161133, "kl": 0.08400401845574379, "learning_rate": 6.819348298638839e-09, "loss": -0.051, "num_tokens": 15164.0, "reward": 0.762055516242981, "reward_std": 0.058255478739738464, "rewards/concensus_correctness_reward_func/mean": 0.0, "rewards/concensus_correctness_reward_func/std": 0.0, "rewards/consensus_reward_func/mean": 0.0, "rewards/consensus_reward_func/std": 0.0, "rewards/cumulative_reward_2/mean": 0.0, "rewards/cumulative_reward_2/std": 0.0, "rewards/final_correctness_reward_func/mean": 0.0, "rewards/final_correctness_reward_func/std": 0.0, "rewards/question_recreation_reward_func/mean": 0.26205551624298096, "rewards/question_recreation_reward_func/std": 0.15907590091228485, "rewards/soft_format_reward_func/mean": 0.0, "rewards/soft_format_reward_func/std": 0.0, "rewards/strict_format_reward_func/mean": 0.0, "rewards/strict_format_reward_func/std": 0.0, "rewards/xmlcount_reward_func/mean": 0.5, "rewards/xmlcount_reward_func/std": 0.0, "step": 20 }, { "epoch": 2.2222222222222223, "step": 20, "total_flos": 0.0, "train_loss": -0.04282761747017503, "train_runtime": 6256.5978, "train_samples_per_second": 0.013, "train_steps_per_second": 0.003 } ], "logging_steps": 1, "max_steps": 20, "num_input_tokens_seen": 15164, "num_train_epochs": 3, "save_steps": 25, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": true, "should_training_stop": true }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 2, "trial_name": null, "trial_params": null }