{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 19.666666666666668, "eval_steps": 500, "global_step": 20, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 32.0, "completions/max_terminated_length": 32.0, "completions/mean_length": 28.33333396911621, "completions/mean_terminated_length": 28.33333396911621, "completions/min_length": 25.0, "completions/min_terminated_length": 25.0, "epoch": 0.6666666666666666, "grad_norm": 23.18532371520996, "kl": 0.0, "learning_rate": 0.0, "loss": -0.0239, "num_tokens": 628.0, "reward": 1.2813235521316528, "reward_std": 0.008508604019880295, "rewards/concensus_correctness_reward_func/mean": 0.0, "rewards/concensus_correctness_reward_func/std": 0.0, "rewards/consensus_reward_func/mean": 0.0, "rewards/consensus_reward_func/std": 0.0, "rewards/cumulative_reward_2/mean": 0.0, "rewards/cumulative_reward_2/std": 0.0, "rewards/final_correctness_reward_func/mean": 1.0, "rewards/final_correctness_reward_func/std": 1.154700517654419, "rewards/question_recreation_reward_func/mean": 0.03132347762584686, "rewards/question_recreation_reward_func/std": 0.008864727802574635, "rewards/soft_format_reward_func/mean": 0.0, "rewards/soft_format_reward_func/std": 0.0, "rewards/strict_format_reward_func/mean": 0.0, "rewards/strict_format_reward_func/std": 0.0, "rewards/xmlcount_reward_func/mean": 0.25, "rewards/xmlcount_reward_func/std": 0.28867512941360474, "step": 1 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 29.0, "completions/max_terminated_length": 29.0, "completions/mean_length": 23.0, "completions/mean_terminated_length": 23.0, "completions/min_length": 15.0, "completions/min_terminated_length": 15.0, "epoch": 1.6666666666666665, "grad_norm": 28.710390090942383, "kl": 0.0, "learning_rate": 1e-06, "loss": 0.1115, "num_tokens": 1234.0, "reward": 1.284814715385437, "reward_std": 0.005810028873383999, "rewards/concensus_correctness_reward_func/mean": 0.0, "rewards/concensus_correctness_reward_func/std": 0.0, "rewards/consensus_reward_func/mean": 0.0, "rewards/consensus_reward_func/std": 0.0, "rewards/cumulative_reward_2/mean": 0.0, "rewards/cumulative_reward_2/std": 0.0, "rewards/final_correctness_reward_func/mean": 1.0, "rewards/final_correctness_reward_func/std": 1.154700517654419, "rewards/question_recreation_reward_func/mean": 0.034814756363630295, "rewards/question_recreation_reward_func/std": 0.024560028687119484, "rewards/soft_format_reward_func/mean": 0.0, "rewards/soft_format_reward_func/std": 0.0, "rewards/strict_format_reward_func/mean": 0.0, "rewards/strict_format_reward_func/std": 0.0, "rewards/xmlcount_reward_func/mean": 0.25, "rewards/xmlcount_reward_func/std": 0.28867512941360474, "step": 2 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 37.0, "completions/max_terminated_length": 37.0, "completions/mean_length": 33.66666793823242, "completions/mean_terminated_length": 33.66666793823242, "completions/min_length": 32.0, "completions/min_terminated_length": 32.0, "epoch": 2.6666666666666665, "grad_norm": 10.300809860229492, "kl": 0.010776982642710209, "learning_rate": 9.931806517013612e-07, "loss": 0.0224, "num_tokens": 1889.0, "reward": 1.2809349298477173, "reward_std": 0.002600151114165783, "rewards/concensus_correctness_reward_func/mean": 0.0, "rewards/concensus_correctness_reward_func/std": 0.0, "rewards/consensus_reward_func/mean": 0.0, "rewards/consensus_reward_func/std": 0.0, "rewards/cumulative_reward_2/mean": 0.0, "rewards/cumulative_reward_2/std": 0.0, "rewards/final_correctness_reward_func/mean": 1.0, "rewards/final_correctness_reward_func/std": 1.154700517654419, "rewards/question_recreation_reward_func/mean": 0.030935000628232956, "rewards/question_recreation_reward_func/std": 0.005902186036109924, "rewards/soft_format_reward_func/mean": 0.0, "rewards/soft_format_reward_func/std": 0.0, "rewards/strict_format_reward_func/mean": 0.0, "rewards/strict_format_reward_func/std": 0.0, "rewards/xmlcount_reward_func/mean": 0.25, "rewards/xmlcount_reward_func/std": 0.28867512941360474, "step": 3 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 32.0, "completions/max_terminated_length": 32.0, "completions/mean_length": 18.33333396911621, "completions/mean_terminated_length": 18.33333396911621, "completions/min_length": 10.0, "completions/min_terminated_length": 10.0, "epoch": 3.6666666666666665, "grad_norm": 34.62272262573242, "kl": 0.16381593234837055, "learning_rate": 9.729086208503173e-07, "loss": -0.0388, "num_tokens": 2488.0, "reward": 1.2828543186187744, "reward_std": 0.002999177435413003, "rewards/concensus_correctness_reward_func/mean": 0.0, "rewards/concensus_correctness_reward_func/std": 0.0, "rewards/consensus_reward_func/mean": 0.0, "rewards/consensus_reward_func/std": 0.0, "rewards/cumulative_reward_2/mean": 0.0, "rewards/cumulative_reward_2/std": 0.0, "rewards/final_correctness_reward_func/mean": 1.0, "rewards/final_correctness_reward_func/std": 1.154700517654419, "rewards/question_recreation_reward_func/mean": 0.032854292541742325, "rewards/question_recreation_reward_func/std": 0.01772441901266575, "rewards/soft_format_reward_func/mean": 0.0, "rewards/soft_format_reward_func/std": 0.0, "rewards/strict_format_reward_func/mean": 0.0, "rewards/strict_format_reward_func/std": 0.0, "rewards/xmlcount_reward_func/mean": 0.25, "rewards/xmlcount_reward_func/std": 0.28867512941360474, "step": 4 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 33.0, "completions/max_terminated_length": 33.0, "completions/mean_length": 25.0, "completions/mean_terminated_length": 25.0, "completions/min_length": 14.0, "completions/min_terminated_length": 14.0, "epoch": 4.666666666666667, "grad_norm": 28.476211547851562, "kl": 0.10297379642724991, "learning_rate": 9.397368756032444e-07, "loss": 0.1969, "num_tokens": 3127.0, "reward": 0.04856175556778908, "reward_std": 0.0062292092479765415, "rewards/concensus_correctness_reward_func/mean": 0.0, "rewards/concensus_correctness_reward_func/std": 0.0, "rewards/consensus_reward_func/mean": 0.0, "rewards/consensus_reward_func/std": 0.0, "rewards/cumulative_reward_2/mean": 0.0, "rewards/cumulative_reward_2/std": 0.0, "rewards/final_correctness_reward_func/mean": 0.0, "rewards/final_correctness_reward_func/std": 0.0, "rewards/question_recreation_reward_func/mean": 0.04856175556778908, "rewards/question_recreation_reward_func/std": 0.011551093310117722, "rewards/soft_format_reward_func/mean": 0.0, "rewards/soft_format_reward_func/std": 0.0, "rewards/strict_format_reward_func/mean": 0.0, "rewards/strict_format_reward_func/std": 0.0, "rewards/xmlcount_reward_func/mean": 0.0, "rewards/xmlcount_reward_func/std": 0.0, "step": 5 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 32.0, "completions/max_terminated_length": 32.0, "completions/mean_length": 24.33333396911621, "completions/mean_terminated_length": 24.33333396911621, "completions/min_length": 16.0, "completions/min_terminated_length": 16.0, "epoch": 5.666666666666667, "grad_norm": 29.99322509765625, "kl": 0.06120647024363279, "learning_rate": 8.945702546981968e-07, "loss": -0.0746, "num_tokens": 3744.0, "reward": 1.2867157459259033, "reward_std": 0.007329343352466822, "rewards/concensus_correctness_reward_func/mean": 0.0, "rewards/concensus_correctness_reward_func/std": 0.0, "rewards/consensus_reward_func/mean": 0.0, "rewards/consensus_reward_func/std": 0.0, "rewards/cumulative_reward_2/mean": 0.0, "rewards/cumulative_reward_2/std": 0.0, "rewards/final_correctness_reward_func/mean": 1.0, "rewards/final_correctness_reward_func/std": 1.154700517654419, "rewards/question_recreation_reward_func/mean": 0.03671575337648392, "rewards/question_recreation_reward_func/std": 0.023405639454722404, "rewards/soft_format_reward_func/mean": 0.0, "rewards/soft_format_reward_func/std": 0.0, "rewards/strict_format_reward_func/mean": 0.0, "rewards/strict_format_reward_func/std": 0.0, "rewards/xmlcount_reward_func/mean": 0.25, "rewards/xmlcount_reward_func/std": 0.28867512941360474, "step": 6 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 49.0, "completions/max_terminated_length": 49.0, "completions/mean_length": 24.0, "completions/mean_terminated_length": 24.0, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "epoch": 6.666666666666667, "grad_norm": 28.319862365722656, "kl": 0.1611136607825756, "learning_rate": 8.386407858128706e-07, "loss": -0.2244, "num_tokens": 4343.0, "reward": 0.026412304490804672, "reward_std": 0.013399245217442513, "rewards/concensus_correctness_reward_func/mean": 0.0, "rewards/concensus_correctness_reward_func/std": 0.0, "rewards/consensus_reward_func/mean": 0.0, "rewards/consensus_reward_func/std": 0.0, "rewards/cumulative_reward_2/mean": 0.0, "rewards/cumulative_reward_2/std": 0.0, "rewards/final_correctness_reward_func/mean": 0.0, "rewards/final_correctness_reward_func/std": 0.0, "rewards/question_recreation_reward_func/mean": 0.026412304490804672, "rewards/question_recreation_reward_func/std": 0.013973159715533257, "rewards/soft_format_reward_func/mean": 0.0, "rewards/soft_format_reward_func/std": 0.0, "rewards/strict_format_reward_func/mean": 0.0, "rewards/strict_format_reward_func/std": 0.0, "rewards/xmlcount_reward_func/mean": 0.0, "rewards/xmlcount_reward_func/std": 0.0, "step": 7 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 32.0, "completions/max_terminated_length": 32.0, "completions/mean_length": 29.0, "completions/mean_terminated_length": 29.0, "completions/min_length": 23.0, "completions/min_terminated_length": 23.0, "epoch": 7.666666666666667, "grad_norm": 20.72979164123535, "kl": 0.054506564512848854, "learning_rate": 7.734740790612136e-07, "loss": -0.0114, "num_tokens": 4967.0, "reward": 1.2918072938919067, "reward_std": 0.0005663956981152296, "rewards/concensus_correctness_reward_func/mean": 0.0, "rewards/concensus_correctness_reward_func/std": 0.0, "rewards/consensus_reward_func/mean": 0.0, "rewards/consensus_reward_func/std": 0.0, "rewards/cumulative_reward_2/mean": 0.0, "rewards/cumulative_reward_2/std": 0.0, "rewards/final_correctness_reward_func/mean": 1.0, "rewards/final_correctness_reward_func/std": 1.154700517654419, "rewards/question_recreation_reward_func/mean": 0.04180721938610077, "rewards/question_recreation_reward_func/std": 0.00750124454498291, "rewards/soft_format_reward_func/mean": 0.0, "rewards/soft_format_reward_func/std": 0.0, "rewards/strict_format_reward_func/mean": 0.0, "rewards/strict_format_reward_func/std": 0.0, "rewards/xmlcount_reward_func/mean": 0.25, "rewards/xmlcount_reward_func/std": 0.28867512941360474, "step": 8 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 32.0, "completions/max_terminated_length": 32.0, "completions/mean_length": 16.66666603088379, "completions/mean_terminated_length": 16.66666603088379, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "epoch": 8.666666666666666, "grad_norm": 16.976764678955078, "kl": 0.4037783732637763, "learning_rate": 7.008477123264847e-07, "loss": 0.0162, "num_tokens": 5561.0, "reward": 1.832707405090332, "reward_std": 0.7097353935241699, "rewards/concensus_correctness_reward_func/mean": 0.0, "rewards/concensus_correctness_reward_func/std": 0.0, "rewards/consensus_reward_func/mean": 0.0, "rewards/consensus_reward_func/std": 0.0, "rewards/cumulative_reward_2/mean": 0.0, "rewards/cumulative_reward_2/std": 0.0, "rewards/final_correctness_reward_func/mean": 1.5, "rewards/final_correctness_reward_func/std": 1.0, "rewards/question_recreation_reward_func/mean": 0.02020736038684845, "rewards/question_recreation_reward_func/std": 0.01928129233419895, "rewards/soft_format_reward_func/mean": 0.0, "rewards/soft_format_reward_func/std": 0.0, "rewards/strict_format_reward_func/mean": 0.0, "rewards/strict_format_reward_func/std": 0.0, "rewards/xmlcount_reward_func/mean": 0.3125, "rewards/xmlcount_reward_func/std": 0.21650634706020355, "step": 9 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 32.0, "completions/max_terminated_length": 32.0, "completions/mean_length": 29.33333396911621, "completions/mean_terminated_length": 29.33333396911621, "completions/min_length": 24.0, "completions/min_terminated_length": 24.0, "epoch": 9.666666666666666, "grad_norm": 27.643352508544922, "kl": 0.14844292961061, "learning_rate": 6.227427435703995e-07, "loss": 0.0466, "num_tokens": 6180.0, "reward": 1.301353096961975, "reward_std": 0.004389901179820299, "rewards/concensus_correctness_reward_func/mean": 0.0, "rewards/concensus_correctness_reward_func/std": 0.0, "rewards/consensus_reward_func/mean": 0.0, "rewards/consensus_reward_func/std": 0.0, "rewards/cumulative_reward_2/mean": 0.0, "rewards/cumulative_reward_2/std": 0.0, "rewards/final_correctness_reward_func/mean": 1.0, "rewards/final_correctness_reward_func/std": 1.154700517654419, "rewards/question_recreation_reward_func/mean": 0.05135301500558853, "rewards/question_recreation_reward_func/std": 0.019177274778485298, "rewards/soft_format_reward_func/mean": 0.0, "rewards/soft_format_reward_func/std": 0.0, "rewards/strict_format_reward_func/mean": 0.0, "rewards/strict_format_reward_func/std": 0.0, "rewards/xmlcount_reward_func/mean": 0.25, "rewards/xmlcount_reward_func/std": 0.28867512941360474, "step": 10 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 32.0, "completions/max_terminated_length": 32.0, "completions/mean_length": 27.0, "completions/mean_terminated_length": 27.0, "completions/min_length": 22.0, "completions/min_terminated_length": 22.0, "epoch": 10.666666666666666, "grad_norm": 23.045936584472656, "kl": 0.12309377361088991, "learning_rate": 5.412896727361662e-07, "loss": -0.0306, "num_tokens": 6805.0, "reward": 1.2927604913711548, "reward_std": 0.0032423806842416525, "rewards/concensus_correctness_reward_func/mean": 0.0, "rewards/concensus_correctness_reward_func/std": 0.0, "rewards/consensus_reward_func/mean": 0.0, "rewards/consensus_reward_func/std": 0.0, "rewards/cumulative_reward_2/mean": 0.0, "rewards/cumulative_reward_2/std": 0.0, "rewards/final_correctness_reward_func/mean": 1.0, "rewards/final_correctness_reward_func/std": 1.154700517654419, "rewards/question_recreation_reward_func/mean": 0.04276055097579956, "rewards/question_recreation_reward_func/std": 0.02906481921672821, "rewards/soft_format_reward_func/mean": 0.0, "rewards/soft_format_reward_func/std": 0.0, "rewards/strict_format_reward_func/mean": 0.0, "rewards/strict_format_reward_func/std": 0.0, "rewards/xmlcount_reward_func/mean": 0.25, "rewards/xmlcount_reward_func/std": 0.28867512941360474, "step": 11 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 34.0, "completions/max_terminated_length": 34.0, "completions/mean_length": 28.33333396911621, "completions/mean_terminated_length": 28.33333396911621, "completions/min_length": 17.0, "completions/min_terminated_length": 17.0, "epoch": 11.666666666666666, "grad_norm": 24.045764923095703, "kl": 0.30065467208623886, "learning_rate": 4.5871032726383385e-07, "loss": -0.1085, "num_tokens": 7437.0, "reward": 0.5449972152709961, "reward_std": 0.7033753395080566, "rewards/concensus_correctness_reward_func/mean": 0.0, "rewards/concensus_correctness_reward_func/std": 0.0, "rewards/consensus_reward_func/mean": 0.0, "rewards/consensus_reward_func/std": 0.0, "rewards/cumulative_reward_2/mean": 0.0, "rewards/cumulative_reward_2/std": 0.0, "rewards/final_correctness_reward_func/mean": 0.5, "rewards/final_correctness_reward_func/std": 1.0, "rewards/question_recreation_reward_func/mean": 0.044997282326221466, "rewards/question_recreation_reward_func/std": 0.011617259122431278, "rewards/soft_format_reward_func/mean": 0.0, "rewards/soft_format_reward_func/std": 0.0, "rewards/strict_format_reward_func/mean": 0.0, "rewards/strict_format_reward_func/std": 0.0, "rewards/xmlcount_reward_func/mean": 0.0, "rewards/xmlcount_reward_func/std": 0.0, "step": 12 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 40.0, "completions/max_terminated_length": 40.0, "completions/mean_length": 36.66666793823242, "completions/mean_terminated_length": 36.66666793823242, "completions/min_length": 32.0, "completions/min_terminated_length": 32.0, "epoch": 12.666666666666666, "grad_norm": 7.944507598876953, "kl": 0.1541438391432166, "learning_rate": 3.772572564296004e-07, "loss": 0.0149, "num_tokens": 8091.0, "reward": 1.7904198169708252, "reward_std": 0.7157720923423767, "rewards/concensus_correctness_reward_func/mean": 0.0, "rewards/concensus_correctness_reward_func/std": 0.0, "rewards/consensus_reward_func/mean": 0.0, "rewards/consensus_reward_func/std": 0.0, "rewards/cumulative_reward_2/mean": 0.0, "rewards/cumulative_reward_2/std": 0.0, "rewards/final_correctness_reward_func/mean": 1.5, "rewards/final_correctness_reward_func/std": 1.0, "rewards/question_recreation_reward_func/mean": 0.03316996619105339, "rewards/question_recreation_reward_func/std": 0.0044271741062402725, "rewards/soft_format_reward_func/mean": 0.0, "rewards/soft_format_reward_func/std": 0.0, "rewards/strict_format_reward_func/mean": 0.0, "rewards/strict_format_reward_func/std": 0.0, "rewards/xmlcount_reward_func/mean": 0.25725001096725464, "rewards/xmlcount_reward_func/std": 0.2805534899234772, "step": 13 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 335.0, "completions/max_terminated_length": 335.0, "completions/mean_length": 135.6666717529297, "completions/mean_terminated_length": 135.6666717529297, "completions/min_length": 32.0, "completions/min_terminated_length": 32.0, "epoch": 13.666666666666666, "grad_norm": 7.2156081199646, "kl": 0.07004917785525322, "learning_rate": 2.9915228767351535e-07, "loss": 0.2809, "num_tokens": 9042.0, "reward": 1.7817018032073975, "reward_std": 0.7197564244270325, "rewards/concensus_correctness_reward_func/mean": 0.0, "rewards/concensus_correctness_reward_func/std": 0.0, "rewards/consensus_reward_func/mean": 0.0, "rewards/consensus_reward_func/std": 0.0, "rewards/cumulative_reward_2/mean": 0.0, "rewards/cumulative_reward_2/std": 0.0, "rewards/final_correctness_reward_func/mean": 1.5, "rewards/final_correctness_reward_func/std": 1.0, "rewards/question_recreation_reward_func/mean": 0.02720182202756405, "rewards/question_recreation_reward_func/std": 0.013149062171578407, "rewards/soft_format_reward_func/mean": 0.0, "rewards/soft_format_reward_func/std": 0.0, "rewards/strict_format_reward_func/mean": 0.0, "rewards/strict_format_reward_func/std": 0.0, "rewards/xmlcount_reward_func/mean": 0.25450000166893005, "rewards/xmlcount_reward_func/std": 0.2835742235183716, "step": 14 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 46.0, "completions/max_terminated_length": 46.0, "completions/mean_length": 24.0, "completions/mean_terminated_length": 24.0, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "epoch": 14.666666666666666, "grad_norm": 39.25900650024414, "kl": 0.3023638464510441, "learning_rate": 2.2652592093878665e-07, "loss": 0.1531, "num_tokens": 9664.0, "reward": 0.07192623615264893, "reward_std": 0.028722776100039482, "rewards/concensus_correctness_reward_func/mean": 0.0, "rewards/concensus_correctness_reward_func/std": 0.0, "rewards/consensus_reward_func/mean": 0.0, "rewards/consensus_reward_func/std": 0.0, "rewards/cumulative_reward_2/mean": 0.0, "rewards/cumulative_reward_2/std": 0.0, "rewards/final_correctness_reward_func/mean": 0.0, "rewards/final_correctness_reward_func/std": 0.0, "rewards/question_recreation_reward_func/mean": 0.04067623242735863, "rewards/question_recreation_reward_func/std": 0.020941482856869698, "rewards/soft_format_reward_func/mean": 0.0, "rewards/soft_format_reward_func/std": 0.0, "rewards/strict_format_reward_func/mean": 0.0, "rewards/strict_format_reward_func/std": 0.0, "rewards/xmlcount_reward_func/mean": 0.03125, "rewards/xmlcount_reward_func/std": 0.0625, "step": 15 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 34.0, "completions/max_terminated_length": 34.0, "completions/mean_length": 32.66666793823242, "completions/mean_terminated_length": 32.66666793823242, "completions/min_length": 32.0, "completions/min_terminated_length": 32.0, "epoch": 15.666666666666666, "grad_norm": 24.114459991455078, "kl": 0.2044437825679779, "learning_rate": 1.6135921418712955e-07, "loss": -0.0388, "num_tokens": 10300.0, "reward": 1.2992045879364014, "reward_std": 0.014325941912829876, "rewards/concensus_correctness_reward_func/mean": 0.0, "rewards/concensus_correctness_reward_func/std": 0.0, "rewards/consensus_reward_func/mean": 0.0, "rewards/consensus_reward_func/std": 0.0, "rewards/cumulative_reward_2/mean": 0.0, "rewards/cumulative_reward_2/std": 0.0, "rewards/final_correctness_reward_func/mean": 1.0, "rewards/final_correctness_reward_func/std": 1.154700517654419, "rewards/question_recreation_reward_func/mean": 0.049204543232917786, "rewards/question_recreation_reward_func/std": 0.023023977875709534, "rewards/soft_format_reward_func/mean": 0.0, "rewards/soft_format_reward_func/std": 0.0, "rewards/strict_format_reward_func/mean": 0.0, "rewards/strict_format_reward_func/std": 0.0, "rewards/xmlcount_reward_func/mean": 0.25, "rewards/xmlcount_reward_func/std": 0.28867512941360474, "step": 16 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 44.0, "completions/max_terminated_length": 44.0, "completions/mean_length": 32.66666793823242, "completions/mean_terminated_length": 32.66666793823242, "completions/min_length": 24.0, "completions/min_terminated_length": 24.0, "epoch": 16.666666666666668, "grad_norm": 24.598146438598633, "kl": 0.348018154501915, "learning_rate": 1.0542974530180327e-07, "loss": 0.0332, "num_tokens": 10942.0, "reward": 1.0598185062408447, "reward_std": 0.008537341840565205, "rewards/concensus_correctness_reward_func/mean": 0.0, "rewards/concensus_correctness_reward_func/std": 0.0, "rewards/consensus_reward_func/mean": 0.0, "rewards/consensus_reward_func/std": 0.0, "rewards/cumulative_reward_2/mean": 0.0, "rewards/cumulative_reward_2/std": 0.0, "rewards/final_correctness_reward_func/mean": 1.0, "rewards/final_correctness_reward_func/std": 1.154700517654419, "rewards/question_recreation_reward_func/mean": 0.04881836846470833, "rewards/question_recreation_reward_func/std": 0.01848919875919819, "rewards/soft_format_reward_func/mean": 0.0, "rewards/soft_format_reward_func/std": 0.0, "rewards/strict_format_reward_func/mean": 0.0, "rewards/strict_format_reward_func/std": 0.0, "rewards/xmlcount_reward_func/mean": 0.010999999940395355, "rewards/xmlcount_reward_func/std": 0.020688161253929138, "step": 17 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 45.0, "completions/max_terminated_length": 45.0, "completions/mean_length": 42.0, "completions/mean_terminated_length": 42.0, "completions/min_length": 40.0, "completions/min_terminated_length": 40.0, "epoch": 17.666666666666668, "grad_norm": 20.540817260742188, "kl": 0.3694180101156235, "learning_rate": 6.026312439675551e-08, "loss": -0.0307, "num_tokens": 11638.0, "reward": 1.0319881439208984, "reward_std": 0.002910168608650565, "rewards/concensus_correctness_reward_func/mean": 0.0, "rewards/concensus_correctness_reward_func/std": 0.0, "rewards/consensus_reward_func/mean": 0.0, "rewards/consensus_reward_func/std": 0.0, "rewards/cumulative_reward_2/mean": 0.0, "rewards/cumulative_reward_2/std": 0.0, "rewards/final_correctness_reward_func/mean": 1.0, "rewards/final_correctness_reward_func/std": 1.154700517654419, "rewards/question_recreation_reward_func/mean": 0.029738131910562515, "rewards/question_recreation_reward_func/std": 0.0061986916698515415, "rewards/soft_format_reward_func/mean": 0.0, "rewards/soft_format_reward_func/std": 0.0, "rewards/strict_format_reward_func/mean": 0.0, "rewards/strict_format_reward_func/std": 0.0, "rewards/xmlcount_reward_func/mean": 0.0022499999031424522, "rewards/xmlcount_reward_func/std": 0.006652066949754953, "step": 18 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 41.0, "completions/max_terminated_length": 41.0, "completions/mean_length": 37.0, "completions/mean_terminated_length": 37.0, "completions/min_length": 30.0, "completions/min_terminated_length": 30.0, "epoch": 18.666666666666668, "grad_norm": 27.271289825439453, "kl": 0.34695659577846527, "learning_rate": 2.7091379149682682e-08, "loss": 0.0705, "num_tokens": 12302.0, "reward": 0.532687783241272, "reward_std": 0.7111693024635315, "rewards/concensus_correctness_reward_func/mean": 0.0, "rewards/concensus_correctness_reward_func/std": 0.0, "rewards/consensus_reward_func/mean": 0.0, "rewards/consensus_reward_func/std": 0.0, "rewards/cumulative_reward_2/mean": 0.0, "rewards/cumulative_reward_2/std": 0.0, "rewards/final_correctness_reward_func/mean": 0.5, "rewards/final_correctness_reward_func/std": 1.0, "rewards/question_recreation_reward_func/mean": 0.0281878300011158, "rewards/question_recreation_reward_func/std": 0.005380373448133469, "rewards/soft_format_reward_func/mean": 0.0, "rewards/soft_format_reward_func/std": 0.0, "rewards/strict_format_reward_func/mean": 0.0, "rewards/strict_format_reward_func/std": 0.0, "rewards/xmlcount_reward_func/mean": 0.0044999998062849045, "rewards/xmlcount_reward_func/std": 0.008999999612569809, "step": 19 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 40.0, "completions/max_terminated_length": 40.0, "completions/mean_length": 34.0, "completions/mean_terminated_length": 34.0, "completions/min_length": 30.0, "completions/min_terminated_length": 30.0, "epoch": 19.666666666666668, "grad_norm": 20.589397430419922, "kl": 0.16628471948206425, "learning_rate": 6.819348298638839e-09, "loss": 0.057, "num_tokens": 12948.0, "reward": 1.2811635732650757, "reward_std": 0.014331253245472908, "rewards/concensus_correctness_reward_func/mean": 0.0, "rewards/concensus_correctness_reward_func/std": 0.0, "rewards/consensus_reward_func/mean": 0.0, "rewards/consensus_reward_func/std": 0.0, "rewards/cumulative_reward_2/mean": 0.0, "rewards/cumulative_reward_2/std": 0.0, "rewards/final_correctness_reward_func/mean": 1.0, "rewards/final_correctness_reward_func/std": 1.154700517654419, "rewards/question_recreation_reward_func/mean": 0.031163623556494713, "rewards/question_recreation_reward_func/std": 0.02261289581656456, "rewards/soft_format_reward_func/mean": 0.0, "rewards/soft_format_reward_func/std": 0.0, "rewards/strict_format_reward_func/mean": 0.0, "rewards/strict_format_reward_func/std": 0.0, "rewards/xmlcount_reward_func/mean": 0.25, "rewards/xmlcount_reward_func/std": 0.28867512941360474, "step": 20 }, { "epoch": 19.666666666666668, "step": 20, "total_flos": 0.0, "train_loss": 0.02108212043531239, "train_runtime": 15960.1034, "train_samples_per_second": 0.005, "train_steps_per_second": 0.001 } ], "logging_steps": 1, "max_steps": 20, "num_input_tokens_seen": 12948, "num_train_epochs": 20, "save_steps": 25, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": true, "should_training_stop": true }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 2, "trial_name": null, "trial_params": null }