{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 0.0022222222222222222, "eval_steps": 500, "global_step": 100, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.125, "completions/max_length": 1024.0, "completions/max_terminated_length": 872.0, "completions/mean_length": 624.5625, "completions/mean_terminated_length": 567.5000305175781, "completions/min_length": 209.5, "completions/min_terminated_length": 209.5, "entropy": 1.33274307847023, "epoch": 4.4444444444444447e-05, "frac_reward_zero_std": 0.0, "grad_norm": 4.749985828335076, "kl": 0.0, "learning_rate": 1.6666666666666665e-07, "loss": -0.1614, "num_tokens": 12457.0, "reward": 0.25, "reward_std": 0.4629100561141968, "rewards/equation_reward_func/mean": 0.0, "rewards/equation_reward_func/std": 0.0, "rewards/format_reward_func/mean": 0.25, "rewards/format_reward_func/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 2.5595189332962036, "sampling/importance_sampling_ratio/mean": 0.7684620320796967, "sampling/importance_sampling_ratio/min": 0.14499648660421371, "sampling/sampling_logp_difference/max": 0.5778531432151794, "sampling/sampling_logp_difference/mean": 0.026035048998892307, "step": 2, "step_time": 10.443448680220172 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 903.0, "completions/max_terminated_length": 838.0, "completions/mean_length": 584.0, "completions/mean_terminated_length": 559.2410888671875, "completions/min_length": 262.5, "completions/min_terminated_length": 262.5, "entropy": 1.2995611056685448, "epoch": 8.888888888888889e-05, "frac_reward_zero_std": 0.5, "grad_norm": 4.687210700851442, "kl": 0.0005161362223589094, "learning_rate": 5e-07, "loss": -0.0495, "num_tokens": 24225.0, "reward": 0.1875, "reward_std": 0.2587745785713196, "rewards/equation_reward_func/mean": 0.0, "rewards/equation_reward_func/std": 0.0, "rewards/format_reward_func/mean": 0.1875, "rewards/format_reward_func/std": 0.25877460837364197, "sampling/importance_sampling_ratio/max": 2.0775018334388733, "sampling/importance_sampling_ratio/mean": 0.982254296541214, "sampling/importance_sampling_ratio/min": 0.0867920070886612, "sampling/sampling_logp_difference/max": 0.41237497329711914, "sampling/sampling_logp_difference/mean": 0.025452900677919388, "step": 4, "step_time": 8.720533160958439 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 894.5, "completions/max_terminated_length": 748.0, "completions/mean_length": 544.9375, "completions/mean_terminated_length": 512.7946472167969, "completions/min_length": 237.5, "completions/min_terminated_length": 237.5, "entropy": 1.1739214062690735, "epoch": 0.00013333333333333334, "frac_reward_zero_std": 0.0, "grad_norm": 1.7835863650793127, "kl": 0.0004755087238663691, "learning_rate": 4.994757065594279e-07, "loss": 0.1861, "num_tokens": 35360.0, "reward": 0.3125, "reward_std": 0.49022960662841797, "rewards/equation_reward_func/mean": 0.0, "rewards/equation_reward_func/std": 0.0, "rewards/format_reward_func/mean": 0.3125, "rewards/format_reward_func/std": 0.49022963643074036, "sampling/importance_sampling_ratio/max": 1.6567679643630981, "sampling/importance_sampling_ratio/mean": 0.5457935333251953, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 0.5021777153015137, "sampling/sampling_logp_difference/mean": 0.023564884439110756, "step": 6, "step_time": 8.810739000793546 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 981.0, "completions/max_terminated_length": 981.0, "completions/mean_length": 519.5625, "completions/mean_terminated_length": 519.5625, "completions/min_length": 211.0, "completions/min_terminated_length": 211.0, "entropy": 1.199426531791687, "epoch": 0.00017777777777777779, "frac_reward_zero_std": 0.0, "grad_norm": 3.2845524174720118, "kl": 0.0006594761198357446, "learning_rate": 4.979050253066063e-07, "loss": -0.0057, "num_tokens": 46153.0, "reward": 0.5, "reward_std": 0.5345224738121033, "rewards/equation_reward_func/mean": 0.0, "rewards/equation_reward_func/std": 0.0, "rewards/format_reward_func/mean": 0.5, "rewards/format_reward_func/std": 0.5345224738121033, "sampling/importance_sampling_ratio/max": 1.4953356981277466, "sampling/importance_sampling_ratio/mean": 0.7003053724765778, "sampling/importance_sampling_ratio/min": 0.10495343431830406, "sampling/sampling_logp_difference/max": 0.3991909921169281, "sampling/sampling_logp_difference/mean": 0.024509839713573456, "step": 8, "step_time": 9.232346954755485 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 806.5, "completions/max_terminated_length": 756.0, "completions/mean_length": 432.4375, "completions/mean_terminated_length": 395.0089416503906, "completions/min_length": 218.5, "completions/min_terminated_length": 218.5, "entropy": 1.1880374103784561, "epoch": 0.00022222222222222223, "frac_reward_zero_std": 0.0, "grad_norm": 3.640790667150943, "kl": 0.0004793386397068389, "learning_rate": 4.952945442245597e-07, "loss": 0.2931, "num_tokens": 55480.0, "reward": 0.375, "reward_std": 0.6208146214485168, "rewards/equation_reward_func/mean": 0.1875, "rewards/equation_reward_func/std": 0.408231720328331, "rewards/format_reward_func/mean": 0.1875, "rewards/format_reward_func/std": 0.408231720328331, "sampling/importance_sampling_ratio/max": 1.863376498222351, "sampling/importance_sampling_ratio/mean": 1.0063321888446808, "sampling/importance_sampling_ratio/min": 0.1957462728023529, "sampling/sampling_logp_difference/max": 0.37483203411102295, "sampling/sampling_logp_difference/mean": 0.023145501501858234, "step": 10, "step_time": 8.811809529666789 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 640.5, "completions/max_terminated_length": 640.5, "completions/mean_length": 423.5625, "completions/mean_terminated_length": 423.5625, "completions/min_length": 180.0, "completions/min_terminated_length": 180.0, "entropy": 0.9937291070818901, "epoch": 0.0002666666666666667, "frac_reward_zero_std": 0.0, "grad_norm": 2.702686109757061, "kl": 0.0006009896087562083, "learning_rate": 4.916552125781528e-07, "loss": 0.0342, "num_tokens": 64697.0, "reward": 0.5625, "reward_std": 0.5518900156021118, "rewards/equation_reward_func/mean": 0.0625, "rewards/equation_reward_func/std": 0.1767766922712326, "rewards/format_reward_func/mean": 0.5, "rewards/format_reward_func/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 1.8338406682014465, "sampling/importance_sampling_ratio/mean": 0.8249517679214478, "sampling/importance_sampling_ratio/min": 0.12194083631038666, "sampling/sampling_logp_difference/max": 0.4249380826950073, "sampling/sampling_logp_difference/mean": 0.02203766629099846, "step": 12, "step_time": 8.034374623908661 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 870.5, "completions/max_terminated_length": 682.5, "completions/mean_length": 420.1875, "completions/mean_terminated_length": 376.9464416503906, "completions/min_length": 142.5, "completions/min_terminated_length": 142.5, "entropy": 1.1723689064383507, "epoch": 0.0003111111111111111, "frac_reward_zero_std": 0.0, "grad_norm": 1.3029004457455653, "kl": 0.0009138785835602903, "learning_rate": 4.870022949890676e-07, "loss": 0.0623, "num_tokens": 73836.0, "reward": 0.4375, "reward_std": 0.636739045381546, "rewards/equation_reward_func/mean": 0.1875, "rewards/equation_reward_func/std": 0.408231720328331, "rewards/format_reward_func/mean": 0.25, "rewards/format_reward_func/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 0.8718301355838776, "sampling/importance_sampling_ratio/mean": 0.46648962795734406, "sampling/importance_sampling_ratio/min": 0.07531017251312733, "sampling/sampling_logp_difference/max": 0.8265839219093323, "sampling/sampling_logp_difference/mean": 0.024566035717725754, "step": 14, "step_time": 8.633461503428407 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 664.0, "completions/max_terminated_length": 664.0, "completions/mean_length": 471.0, "completions/mean_terminated_length": 471.0, "completions/min_length": 241.5, "completions/min_terminated_length": 241.5, "entropy": 1.0899769812822342, "epoch": 0.00035555555555555557, "frac_reward_zero_std": 0.5, "grad_norm": 0.0010745967865654204, "kl": 0.000946686575844069, "learning_rate": 4.81355307410676e-07, "loss": 0.0858, "num_tokens": 83804.0, "reward": 0.125, "reward_std": 0.2314550280570984, "rewards/equation_reward_func/mean": 0.0, "rewards/equation_reward_func/std": 0.0, "rewards/format_reward_func/mean": 0.125, "rewards/format_reward_func/std": 0.2314550280570984, "sampling/importance_sampling_ratio/max": 1.2488415241241455, "sampling/importance_sampling_ratio/mean": 0.6085900068283081, "sampling/importance_sampling_ratio/min": 0.10226996056735516, "sampling/sampling_logp_difference/max": 0.4703601598739624, "sampling/sampling_logp_difference/mean": 0.023032911121845245, "step": 16, "step_time": 8.332381021697074 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 856.5, "completions/max_terminated_length": 856.5, "completions/mean_length": 538.625, "completions/mean_terminated_length": 538.625, "completions/min_length": 249.5, "completions/min_terminated_length": 249.5, "entropy": 1.2622679881751537, "epoch": 0.0004, "frac_reward_zero_std": 0.0, "grad_norm": 2.971699889643826, "kl": 0.001047886358719552, "learning_rate": 4.747379352713488e-07, "loss": -0.0704, "num_tokens": 94886.0, "reward": 0.6875, "reward_std": 0.5876962244510651, "rewards/equation_reward_func/mean": 0.0625, "rewards/equation_reward_func/std": 0.1767766922712326, "rewards/format_reward_func/mean": 0.625, "rewards/format_reward_func/std": 0.49871626496315, "sampling/importance_sampling_ratio/max": 1.9497967958450317, "sampling/importance_sampling_ratio/mean": 0.8915638625621796, "sampling/importance_sampling_ratio/min": 0.22629090957343578, "sampling/sampling_logp_difference/max": 0.42359161376953125, "sampling/sampling_logp_difference/mean": 0.02525496855378151, "step": 18, "step_time": 8.536863426212221 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 723.0, "completions/max_terminated_length": 723.0, "completions/mean_length": 453.875, "completions/mean_terminated_length": 453.875, "completions/min_length": 149.0, "completions/min_terminated_length": 149.0, "entropy": 1.1858837865293026, "epoch": 0.00044444444444444447, "frac_reward_zero_std": 0.0, "grad_norm": 3.3370582534401416, "kl": 0.001601253719854867, "learning_rate": 4.6717793412953776e-07, "loss": -0.0119, "num_tokens": 104620.0, "reward": 0.4375, "reward_std": 0.5260358154773712, "rewards/equation_reward_func/mean": 0.0, "rewards/equation_reward_func/std": 0.0, "rewards/format_reward_func/mean": 0.4375, "rewards/format_reward_func/std": 0.5260358452796936, "sampling/importance_sampling_ratio/max": 1.5436645448207855, "sampling/importance_sampling_ratio/mean": 0.8180775046348572, "sampling/importance_sampling_ratio/min": 0.04623246192932129, "sampling/sampling_logp_difference/max": 0.39241665601730347, "sampling/sampling_logp_difference/mean": 0.02441485971212387, "step": 20, "step_time": 8.402821157709695 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 803.5, "completions/max_terminated_length": 803.5, "completions/mean_length": 548.375, "completions/mean_terminated_length": 548.375, "completions/min_length": 346.0, "completions/min_terminated_length": 346.0, "entropy": 1.3530792817473412, "epoch": 0.0004888888888888889, "frac_reward_zero_std": 0.0, "grad_norm": 1.5438846441337637, "kl": 0.0017551442215335555, "learning_rate": 4.5870701325731773e-07, "loss": 0.0066, "num_tokens": 115874.0, "reward": 0.1875, "reward_std": 0.408231720328331, "rewards/equation_reward_func/mean": 0.0, "rewards/equation_reward_func/std": 0.0, "rewards/format_reward_func/mean": 0.1875, "rewards/format_reward_func/std": 0.408231720328331, "sampling/importance_sampling_ratio/max": 2.0694342851638794, "sampling/importance_sampling_ratio/mean": 0.7663778364658356, "sampling/importance_sampling_ratio/min": 0.07287665084004402, "sampling/sampling_logp_difference/max": 0.4795522689819336, "sampling/sampling_logp_difference/mean": 0.026419796980917454, "step": 22, "step_time": 8.40855041577015 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 727.0, "completions/max_terminated_length": 727.0, "completions/mean_length": 427.1875, "completions/mean_terminated_length": 427.1875, "completions/min_length": 192.5, "completions/min_terminated_length": 192.5, "entropy": 1.1948794089257717, "epoch": 0.0005333333333333334, "frac_reward_zero_std": 0.0, "grad_norm": 1.8019870210040545, "kl": 0.00165552277394454, "learning_rate": 4.4936070264068016e-07, "loss": -0.0724, "num_tokens": 125149.0, "reward": 0.5, "reward_std": 0.5850084125995636, "rewards/equation_reward_func/mean": 0.125, "rewards/equation_reward_func/std": 0.2314550280570984, "rewards/format_reward_func/mean": 0.375, "rewards/format_reward_func/std": 0.49871626496315, "sampling/importance_sampling_ratio/max": 1.5116711854934692, "sampling/importance_sampling_ratio/mean": 0.5533610284328461, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 0.5177854299545288, "sampling/sampling_logp_difference/mean": 0.025090673007071018, "step": 24, "step_time": 8.48335952940397 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 967.0, "completions/max_terminated_length": 941.5, "completions/mean_length": 545.875, "completions/mean_terminated_length": 517.6160888671875, "completions/min_length": 226.0, "completions/min_terminated_length": 226.0, "entropy": 1.184289637953043, "epoch": 0.0005777777777777778, "frac_reward_zero_std": 0.0, "grad_norm": 2.829245155965375, "kl": 0.0043001044177799486, "learning_rate": 4.391782039544238e-07, "loss": 0.0465, "num_tokens": 136355.0, "reward": 0.5, "reward_std": 0.5175491571426392, "rewards/equation_reward_func/mean": 0.0, "rewards/equation_reward_func/std": 0.0, "rewards/format_reward_func/mean": 0.5, "rewards/format_reward_func/std": 0.5175492167472839, "sampling/importance_sampling_ratio/max": 1.729424774646759, "sampling/importance_sampling_ratio/mean": 0.8748546242713928, "sampling/importance_sampling_ratio/min": 0.19289711862802505, "sampling/sampling_logp_difference/max": 0.533031702041626, "sampling/sampling_logp_difference/mean": 0.023543622344732285, "step": 26, "step_time": 9.136462694499642 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 976.5, "completions/max_terminated_length": 855.5, "completions/mean_length": 524.75, "completions/mean_terminated_length": 494.3035888671875, "completions/min_length": 249.5, "completions/min_terminated_length": 249.5, "entropy": 1.2454889379441738, "epoch": 0.0006222222222222223, "frac_reward_zero_std": 0.0, "grad_norm": 2.0966414039490338, "kl": 0.004009995092928875, "learning_rate": 4.282022261367073e-07, "loss": -0.0593, "num_tokens": 147215.0, "reward": 0.5625, "reward_std": 0.49022960662841797, "rewards/equation_reward_func/mean": 0.0, "rewards/equation_reward_func/std": 0.0, "rewards/format_reward_func/mean": 0.5625, "rewards/format_reward_func/std": 0.49022963643074036, "sampling/importance_sampling_ratio/max": 2.0301511883735657, "sampling/importance_sampling_ratio/mean": 0.6258620917797089, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 0.4906817674636841, "sampling/sampling_logp_difference/mean": 0.025028634816408157, "step": 28, "step_time": 9.194792822585441 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 981.0, "completions/max_terminated_length": 818.5, "completions/mean_length": 529.375, "completions/mean_terminated_length": 496.3035888671875, "completions/min_length": 237.0, "completions/min_terminated_length": 237.0, "entropy": 1.3521863892674446, "epoch": 0.0006666666666666666, "frac_reward_zero_std": 0.0, "grad_norm": 2.6396233251107017, "kl": 0.002601242988021113, "learning_rate": 4.1647880625292027e-07, "loss": -0.2112, "num_tokens": 158125.0, "reward": 0.5, "reward_std": 0.5175491571426392, "rewards/equation_reward_func/mean": 0.0, "rewards/equation_reward_func/std": 0.0, "rewards/format_reward_func/mean": 0.5, "rewards/format_reward_func/std": 0.5175492167472839, "sampling/importance_sampling_ratio/max": 1.740391492843628, "sampling/importance_sampling_ratio/mean": 0.60825514793396, "sampling/importance_sampling_ratio/min": 0.1199931800365448, "sampling/sampling_logp_difference/max": 0.7060830593109131, "sampling/sampling_logp_difference/mean": 0.025156039744615555, "step": 30, "step_time": 9.511650393018499 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 1000.0, "completions/max_terminated_length": 783.0, "completions/mean_length": 504.625, "completions/mean_terminated_length": 460.24107360839844, "completions/min_length": 225.5, "completions/min_terminated_length": 225.5, "entropy": 1.1279298327863216, "epoch": 0.0007111111111111111, "frac_reward_zero_std": 0.0, "grad_norm": 2.3928007543799974, "kl": 0.006199299117724877, "learning_rate": 4.040571164002318e-07, "loss": -0.2301, "num_tokens": 168679.0, "reward": 0.5625, "reward_std": 0.49022960662841797, "rewards/equation_reward_func/mean": 0.0, "rewards/equation_reward_func/std": 0.0, "rewards/format_reward_func/mean": 0.5625, "rewards/format_reward_func/std": 0.49022963643074036, "sampling/importance_sampling_ratio/max": 2.283095419406891, "sampling/importance_sampling_ratio/mean": 0.9702154397964478, "sampling/importance_sampling_ratio/min": 0.26252565532922745, "sampling/sampling_logp_difference/max": 0.8464382886886597, "sampling/sampling_logp_difference/mean": 0.022625007666647434, "step": 32, "step_time": 9.397412348655052 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 902.0, "completions/max_terminated_length": 902.0, "completions/mean_length": 513.125, "completions/mean_terminated_length": 513.125, "completions/min_length": 150.0, "completions/min_terminated_length": 150.0, "entropy": 1.3348925039172173, "epoch": 0.0007555555555555555, "frac_reward_zero_std": 0.0, "grad_norm": 3.00057282284152, "kl": 0.0058585846272762865, "learning_rate": 3.909892574627266e-07, "loss": -0.0172, "num_tokens": 179305.0, "reward": 0.6875, "reward_std": 0.6123279631137848, "rewards/equation_reward_func/mean": 0.0625, "rewards/equation_reward_func/std": 0.1767766922712326, "rewards/format_reward_func/mean": 0.625, "rewards/format_reward_func/std": 0.5175492167472839, "sampling/importance_sampling_ratio/max": 2.2137855291366577, "sampling/importance_sampling_ratio/mean": 0.805354505777359, "sampling/importance_sampling_ratio/min": 0.12096437066793442, "sampling/sampling_logp_difference/max": 0.47625434398651123, "sampling/sampling_logp_difference/mean": 0.025196850299835205, "step": 34, "step_time": 8.92305501177907 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 749.0, "completions/max_terminated_length": 749.0, "completions/mean_length": 374.1875, "completions/mean_terminated_length": 374.1875, "completions/min_length": 201.0, "completions/min_terminated_length": 201.0, "entropy": 1.0916240960359573, "epoch": 0.0008, "frac_reward_zero_std": 0.0, "grad_norm": 1.9305443524320012, "kl": 0.005473379431350622, "learning_rate": 3.773300405821908e-07, "loss": 0.1427, "num_tokens": 187748.0, "reward": 0.6875, "reward_std": 0.49022960662841797, "rewards/equation_reward_func/mean": 0.0, "rewards/equation_reward_func/std": 0.0, "rewards/format_reward_func/mean": 0.6875, "rewards/format_reward_func/std": 0.49022963643074036, "sampling/importance_sampling_ratio/max": 1.9894119501113892, "sampling/importance_sampling_ratio/mean": 0.7066371142864227, "sampling/importance_sampling_ratio/min": 0.16429987736046314, "sampling/sampling_logp_difference/max": 0.34800171852111816, "sampling/sampling_logp_difference/mean": 0.02382123749703169, "step": 36, "step_time": 8.38340774225071 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 743.0, "completions/max_terminated_length": 607.5, "completions/mean_length": 432.875, "completions/mean_terminated_length": 400.0446472167969, "completions/min_length": 193.0, "completions/min_terminated_length": 193.0, "entropy": 1.1879093013703823, "epoch": 0.0008444444444444444, "frac_reward_zero_std": 0.0, "grad_norm": 1.6618913839693144, "kl": 0.0069876342167845, "learning_rate": 3.6313675726113475e-07, "loss": 0.0062, "num_tokens": 197082.0, "reward": 0.8125, "reward_std": 0.6739883720874786, "rewards/equation_reward_func/mean": 0.125, "rewards/equation_reward_func/std": 0.3535533845424652, "rewards/format_reward_func/mean": 0.6875, "rewards/format_reward_func/std": 0.49022963643074036, "sampling/importance_sampling_ratio/max": 1.6417380571365356, "sampling/importance_sampling_ratio/mean": 0.73714280128479, "sampling/importance_sampling_ratio/min": 0.035142697393894196, "sampling/sampling_logp_difference/max": 0.5109619498252869, "sampling/sampling_logp_difference/mean": 0.025707824155688286, "step": 38, "step_time": 8.638400478637777 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 906.0, "completions/max_terminated_length": 906.0, "completions/mean_length": 495.0, "completions/mean_terminated_length": 495.0, "completions/min_length": 255.0, "completions/min_terminated_length": 255.0, "entropy": 1.2622622810304165, "epoch": 0.0008888888888888889, "frac_reward_zero_std": 0.0, "grad_norm": 2.2967943028741558, "kl": 0.005186507463804446, "learning_rate": 3.484689390623218e-07, "loss": -0.1358, "num_tokens": 207442.0, "reward": 0.75, "reward_std": 0.4355512708425522, "rewards/equation_reward_func/mean": 0.0, "rewards/equation_reward_func/std": 0.0, "rewards/format_reward_func/mean": 0.75, "rewards/format_reward_func/std": 0.4355513006448746, "sampling/importance_sampling_ratio/max": 2.0180424451828003, "sampling/importance_sampling_ratio/mean": 0.8196437358856201, "sampling/importance_sampling_ratio/min": 0.11646383255720139, "sampling/sampling_logp_difference/max": 0.4600484371185303, "sampling/sampling_logp_difference/mean": 0.02518580388277769, "step": 40, "step_time": 9.020468398462981 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 748.0, "completions/max_terminated_length": 748.0, "completions/mean_length": 448.6875, "completions/mean_terminated_length": 448.6875, "completions/min_length": 135.5, "completions/min_terminated_length": 135.5, "entropy": 1.1781928315758705, "epoch": 0.0009333333333333333, "frac_reward_zero_std": 0.0, "grad_norm": 3.2445751378816468, "kl": 0.007875571063777898, "learning_rate": 3.3338810791270517e-07, "loss": -0.0195, "num_tokens": 217037.0, "reward": 0.75, "reward_std": 0.6746576428413391, "rewards/equation_reward_func/mean": 0.125, "rewards/equation_reward_func/std": 0.2314550280570984, "rewards/format_reward_func/mean": 0.625, "rewards/format_reward_func/std": 0.49871626496315, "sampling/importance_sampling_ratio/max": 2.228373646736145, "sampling/importance_sampling_ratio/mean": 0.8085304498672485, "sampling/importance_sampling_ratio/min": 0.05368908867239952, "sampling/sampling_logp_difference/max": 0.5465056896209717, "sampling/sampling_logp_difference/mean": 0.024572196416556835, "step": 42, "step_time": 8.777628015144728 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 691.5, "completions/max_terminated_length": 691.5, "completions/mean_length": 433.25, "completions/mean_terminated_length": 433.25, "completions/min_length": 236.5, "completions/min_terminated_length": 236.5, "entropy": 1.2821482792496681, "epoch": 0.0009777777777777777, "frac_reward_zero_std": 0.0, "grad_norm": 5.114077373655992, "kl": 0.0050713238888420165, "learning_rate": 3.179575180590857e-07, "loss": -0.0374, "num_tokens": 226449.0, "reward": 0.8125, "reward_std": 0.5518900156021118, "rewards/equation_reward_func/mean": 0.0625, "rewards/equation_reward_func/std": 0.1767766922712326, "rewards/format_reward_func/mean": 0.75, "rewards/format_reward_func/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 1.9372240900993347, "sampling/importance_sampling_ratio/mean": 1.0048049092292786, "sampling/importance_sampling_ratio/min": 0.06090385094285011, "sampling/sampling_logp_difference/max": 0.4808278977870941, "sampling/sampling_logp_difference/mean": 0.0246611712500453, "step": 44, "step_time": 8.335677679046057 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 855.5, "completions/max_terminated_length": 802.0, "completions/mean_length": 511.3125, "completions/mean_terminated_length": 474.3571472167969, "completions/min_length": 244.5, "completions/min_terminated_length": 244.5, "entropy": 1.1734298951923847, "epoch": 0.0010222222222222223, "frac_reward_zero_std": 0.0, "grad_norm": 3.9815566006792418, "kl": 0.005326784594217315, "learning_rate": 3.022418907578188e-07, "loss": 0.4983, "num_tokens": 237046.0, "reward": 0.625, "reward_std": 0.5175491571426392, "rewards/equation_reward_func/mean": 0.0, "rewards/equation_reward_func/std": 0.0, "rewards/format_reward_func/mean": 0.625, "rewards/format_reward_func/std": 0.5175492167472839, "sampling/importance_sampling_ratio/max": 2.138944089412689, "sampling/importance_sampling_ratio/mean": 1.1211198568344116, "sampling/importance_sampling_ratio/min": 0.320043221116066, "sampling/sampling_logp_difference/max": 0.4653007984161377, "sampling/sampling_logp_difference/mean": 0.023578885942697525, "step": 46, "step_time": 8.643824698519893 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 771.0, "completions/max_terminated_length": 771.0, "completions/mean_length": 439.125, "completions/mean_terminated_length": 439.125, "completions/min_length": 216.0, "completions/min_terminated_length": 216.0, "entropy": 1.2340615428984165, "epoch": 0.0010666666666666667, "frac_reward_zero_std": 0.5, "grad_norm": 4.172865195190822, "kl": 0.004765985926496796, "learning_rate": 2.863071428113726e-07, "loss": 0.0352, "num_tokens": 246552.0, "reward": 0.875, "reward_std": 0.2314550280570984, "rewards/equation_reward_func/mean": 0.0, "rewards/equation_reward_func/std": 0.0, "rewards/format_reward_func/mean": 0.875, "rewards/format_reward_func/std": 0.2314550280570984, "sampling/importance_sampling_ratio/max": 1.7466660737991333, "sampling/importance_sampling_ratio/mean": 0.8979056179523468, "sampling/importance_sampling_ratio/min": 0.14472143352031708, "sampling/sampling_logp_difference/max": 0.3502110242843628, "sampling/sampling_logp_difference/mean": 0.024309751577675343, "step": 48, "step_time": 8.856163574731909 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 791.5, "completions/max_terminated_length": 791.5, "completions/mean_length": 456.375, "completions/mean_terminated_length": 456.375, "completions/min_length": 237.0, "completions/min_terminated_length": 237.0, "entropy": 1.326795369386673, "epoch": 0.0011111111111111111, "frac_reward_zero_std": 0.0, "grad_norm": 1.8629349975816751, "kl": 0.0053678700351156294, "learning_rate": 2.7022011009035107e-07, "loss": -0.0543, "num_tokens": 256334.0, "reward": 0.8125, "reward_std": 0.408231720328331, "rewards/equation_reward_func/mean": 0.0, "rewards/equation_reward_func/std": 0.0, "rewards/format_reward_func/mean": 0.8125, "rewards/format_reward_func/std": 0.408231720328331, "sampling/importance_sampling_ratio/max": 1.5121912360191345, "sampling/importance_sampling_ratio/mean": 0.6738958954811096, "sampling/importance_sampling_ratio/min": 0.018768588081002235, "sampling/sampling_logp_difference/max": 0.4129265546798706, "sampling/sampling_logp_difference/mean": 0.025299090892076492, "step": 50, "step_time": 8.548322821385227 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 778.5, "completions/max_terminated_length": 778.5, "completions/mean_length": 472.5625, "completions/mean_terminated_length": 472.5625, "completions/min_length": 221.0, "completions/min_terminated_length": 221.0, "entropy": 1.2406046949326992, "epoch": 0.0011555555555555555, "frac_reward_zero_std": 0.0, "grad_norm": 1.4813630161379192, "kl": 0.008232236992625985, "learning_rate": 2.540482672006254e-07, "loss": -0.0971, "num_tokens": 266343.0, "reward": 0.625, "reward_std": 0.49871626496315, "rewards/equation_reward_func/mean": 0.0, "rewards/equation_reward_func/std": 0.0, "rewards/format_reward_func/mean": 0.625, "rewards/format_reward_func/std": 0.49871626496315, "sampling/importance_sampling_ratio/max": 0.9217514991760254, "sampling/importance_sampling_ratio/mean": 0.4532146751880646, "sampling/importance_sampling_ratio/min": 0.004407819826155901, "sampling/sampling_logp_difference/max": 1.6009975671768188, "sampling/sampling_logp_difference/mean": 0.025143861770629883, "step": 52, "step_time": 8.712176971603185 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 726.5, "completions/max_terminated_length": 726.5, "completions/mean_length": 438.25, "completions/mean_terminated_length": 438.25, "completions/min_length": 239.5, "completions/min_terminated_length": 239.5, "entropy": 1.098836075514555, "epoch": 0.0012, "frac_reward_zero_std": 0.5, "grad_norm": 0.003079731152243608, "kl": 0.006043522240361199, "learning_rate": 2.37859444471388e-07, "loss": -0.0679, "num_tokens": 275835.0, "reward": 0.9375, "reward_std": 0.1767766922712326, "rewards/equation_reward_func/mean": 0.0, "rewards/equation_reward_func/std": 0.0, "rewards/format_reward_func/mean": 0.9375, "rewards/format_reward_func/std": 0.1767766922712326, "sampling/importance_sampling_ratio/max": 1.760864406824112, "sampling/importance_sampling_ratio/mean": 0.669313445687294, "sampling/importance_sampling_ratio/min": 0.037004563957452774, "sampling/sampling_logp_difference/max": 0.4206712245941162, "sampling/sampling_logp_difference/mean": 0.024013984948396683, "step": 54, "step_time": 8.665698574273847 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 827.5, "completions/max_terminated_length": 827.5, "completions/mean_length": 474.4375, "completions/mean_terminated_length": 474.4375, "completions/min_length": 179.0, "completions/min_terminated_length": 179.0, "entropy": 1.2204781025648117, "epoch": 0.0012444444444444445, "frac_reward_zero_std": 0.0, "grad_norm": 1.5745030120265129, "kl": 0.007742437141132541, "learning_rate": 2.2172154345117894e-07, "loss": -0.1581, "num_tokens": 285890.0, "reward": 0.8125, "reward_std": 0.408231720328331, "rewards/equation_reward_func/mean": 0.0, "rewards/equation_reward_func/std": 0.0, "rewards/format_reward_func/mean": 0.8125, "rewards/format_reward_func/std": 0.408231720328331, "sampling/importance_sampling_ratio/max": 1.5822263956069946, "sampling/importance_sampling_ratio/mean": 0.6425267159938812, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 0.843230128288269, "sampling/sampling_logp_difference/mean": 0.025018153712153435, "step": 56, "step_time": 9.268435405334458 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 800.0, "completions/max_terminated_length": 800.0, "completions/mean_length": 479.5, "completions/mean_terminated_length": 479.5, "completions/min_length": 202.5, "completions/min_terminated_length": 202.5, "entropy": 1.111366856843233, "epoch": 0.001288888888888889, "frac_reward_zero_std": 0.0, "grad_norm": 4.168791141589522, "kl": 0.008615911778178997, "learning_rate": 2.0570225210519433e-07, "loss": -0.0564, "num_tokens": 296010.0, "reward": 1.0625, "reward_std": 0.408231720328331, "rewards/equation_reward_func/mean": 0.125, "rewards/equation_reward_func/std": 0.2314550280570984, "rewards/format_reward_func/mean": 0.9375, "rewards/format_reward_func/std": 0.1767766922712326, "sampling/importance_sampling_ratio/max": 1.7061268091201782, "sampling/importance_sampling_ratio/mean": 0.6005833446979523, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 0.441937193274498, "sampling/sampling_logp_difference/mean": 0.022888539358973503, "step": 58, "step_time": 8.758348444476724 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 712.0, "completions/max_terminated_length": 712.0, "completions/mean_length": 428.0, "completions/mean_terminated_length": 428.0, "completions/min_length": 233.0, "completions/min_terminated_length": 233.0, "entropy": 1.1277053244411945, "epoch": 0.0013333333333333333, "frac_reward_zero_std": 0.0, "grad_norm": 5.575149195352441, "kl": 0.006991666596150026, "learning_rate": 1.8986876090843664e-07, "loss": 0.0453, "num_tokens": 305266.0, "reward": 0.8125, "reward_std": 0.5876962244510651, "rewards/equation_reward_func/mean": 0.125, "rewards/equation_reward_func/std": 0.2314550280570984, "rewards/format_reward_func/mean": 0.6875, "rewards/format_reward_func/std": 0.44403792917728424, "sampling/importance_sampling_ratio/max": 2.0417069792747498, "sampling/importance_sampling_ratio/mean": 0.8298235833644867, "sampling/importance_sampling_ratio/min": 0.06559842079877853, "sampling/sampling_logp_difference/max": 0.3905753046274185, "sampling/sampling_logp_difference/mean": 0.023326802998781204, "step": 60, "step_time": 8.25899032666348 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 975.5, "completions/max_terminated_length": 950.0, "completions/mean_length": 504.625, "completions/mean_terminated_length": 469.02679443359375, "completions/min_length": 218.0, "completions/min_terminated_length": 218.0, "entropy": 1.1242654994130135, "epoch": 0.0013777777777777777, "frac_reward_zero_std": 0.0, "grad_norm": 0.6518752561806848, "kl": 0.006885310140205547, "learning_rate": 1.7428748102551234e-07, "loss": -0.0569, "num_tokens": 315764.0, "reward": 0.8125, "reward_std": 0.408231720328331, "rewards/equation_reward_func/mean": 0.0625, "rewards/equation_reward_func/std": 0.1767766922712326, "rewards/format_reward_func/mean": 0.75, "rewards/format_reward_func/std": 0.4355513006448746, "sampling/importance_sampling_ratio/max": 1.5682951211929321, "sampling/importance_sampling_ratio/mean": 0.6582451313734055, "sampling/importance_sampling_ratio/min": 0.13125959038734436, "sampling/sampling_logp_difference/max": 0.4419562816619873, "sampling/sampling_logp_difference/mean": 0.02288186550140381, "step": 62, "step_time": 9.141117992810905 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 890.5, "completions/max_terminated_length": 890.5, "completions/mean_length": 540.75, "completions/mean_terminated_length": 540.75, "completions/min_length": 214.0, "completions/min_terminated_length": 214.0, "entropy": 1.2960376776754856, "epoch": 0.0014222222222222223, "frac_reward_zero_std": 0.0, "grad_norm": 1.562892780108444, "kl": 0.006526344732264988, "learning_rate": 1.5902376575912814e-07, "loss": 0.0148, "num_tokens": 326848.0, "reward": 0.75, "reward_std": 0.4355512708425522, "rewards/equation_reward_func/mean": 0.0, "rewards/equation_reward_func/std": 0.0, "rewards/format_reward_func/mean": 0.75, "rewards/format_reward_func/std": 0.4355513006448746, "sampling/importance_sampling_ratio/max": 2.478863477706909, "sampling/importance_sampling_ratio/mean": 0.8480255007743835, "sampling/importance_sampling_ratio/min": 0.09445519745349884, "sampling/sampling_logp_difference/max": 0.4862937927246094, "sampling/sampling_logp_difference/mean": 0.025070350617170334, "step": 64, "step_time": 8.837976877926849 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 740.0, "completions/max_terminated_length": 740.0, "completions/mean_length": 394.8125, "completions/mean_terminated_length": 394.8125, "completions/min_length": 167.5, "completions/min_terminated_length": 167.5, "entropy": 1.1651179119944572, "epoch": 0.0014666666666666667, "frac_reward_zero_std": 0.0, "grad_norm": 2.7610832787793433, "kl": 0.009777049403055571, "learning_rate": 1.4414163643562753e-07, "loss": 0.0365, "num_tokens": 335597.0, "reward": 0.875, "reward_std": 0.5940381735563278, "rewards/equation_reward_func/mean": 0.125, "rewards/equation_reward_func/std": 0.2314550280570984, "rewards/format_reward_func/mean": 0.75, "rewards/format_reward_func/std": 0.4355513006448746, "sampling/importance_sampling_ratio/max": 2.0373843908309937, "sampling/importance_sampling_ratio/mean": 0.7534764409065247, "sampling/importance_sampling_ratio/min": 0.07185052335262299, "sampling/sampling_logp_difference/max": 0.6981399059295654, "sampling/sampling_logp_difference/mean": 0.025042645633220673, "step": 66, "step_time": 8.30885543313343 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.125, "completions/max_length": 798.5, "completions/max_terminated_length": 776.5, "completions/mean_length": 574.1875, "completions/mean_terminated_length": 523.375, "completions/min_length": 278.0, "completions/min_terminated_length": 278.0, "entropy": 1.1272502169013023, "epoch": 0.001511111111111111, "frac_reward_zero_std": 0.0, "grad_norm": 2.6380871495100986, "kl": 0.0075112473423359916, "learning_rate": 1.2970351387729872e-07, "loss": -0.1702, "num_tokens": 347264.0, "reward": 0.6875, "reward_std": 0.49022960662841797, "rewards/equation_reward_func/mean": 0.0, "rewards/equation_reward_func/std": 0.0, "rewards/format_reward_func/mean": 0.6875, "rewards/format_reward_func/std": 0.49022963643074036, "sampling/importance_sampling_ratio/max": 1.8942421078681946, "sampling/importance_sampling_ratio/mean": 0.7581468820571899, "sampling/importance_sampling_ratio/min": 0.22752907872200012, "sampling/sampling_logp_difference/max": 0.6634873747825623, "sampling/sampling_logp_difference/mean": 0.022335450164973736, "step": 68, "step_time": 8.51237611565739 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 613.5, "completions/max_terminated_length": 613.5, "completions/mean_length": 412.1875, "completions/mean_terminated_length": 412.1875, "completions/min_length": 220.5, "completions/min_terminated_length": 220.5, "entropy": 1.2611229680478573, "epoch": 0.0015555555555555555, "frac_reward_zero_std": 0.0, "grad_norm": 5.223649574374791, "kl": 0.011163236122229137, "learning_rate": 1.1576995658775404e-07, "loss": -0.1019, "num_tokens": 356299.0, "reward": 0.75, "reward_std": 0.579209566116333, "rewards/equation_reward_func/mean": 0.0625, "rewards/equation_reward_func/std": 0.1767766922712326, "rewards/format_reward_func/mean": 0.6875, "rewards/format_reward_func/std": 0.49022963643074036, "sampling/importance_sampling_ratio/max": 2.2894433736801147, "sampling/importance_sampling_ratio/mean": 0.8129299879074097, "sampling/importance_sampling_ratio/min": 0.15451717376708984, "sampling/sampling_logp_difference/max": 0.3217533826828003, "sampling/sampling_logp_difference/mean": 0.024897821247577667, "step": 70, "step_time": 8.147723604924977 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 785.0, "completions/max_terminated_length": 785.0, "completions/mean_length": 559.6875, "completions/mean_terminated_length": 559.6875, "completions/min_length": 281.0, "completions/min_terminated_length": 281.0, "entropy": 1.2647213600575924, "epoch": 0.0016, "frac_reward_zero_std": 0.0, "grad_norm": 2.061023835100866, "kl": 0.00741576035215985, "learning_rate": 1.0239940674851941e-07, "loss": -0.2195, "num_tokens": 367726.0, "reward": 0.75, "reward_std": 0.4629100561141968, "rewards/equation_reward_func/mean": 0.0, "rewards/equation_reward_func/std": 0.0, "rewards/format_reward_func/mean": 0.75, "rewards/format_reward_func/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 1.8395887613296509, "sampling/importance_sampling_ratio/mean": 0.7119902670383453, "sampling/importance_sampling_ratio/min": 0.05588642507791519, "sampling/sampling_logp_difference/max": 0.40247368812561035, "sampling/sampling_logp_difference/mean": 0.023968717083334923, "step": 72, "step_time": 8.402685680543073 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 915.0, "completions/max_terminated_length": 915.0, "completions/mean_length": 527.25, "completions/mean_terminated_length": 527.25, "completions/min_length": 158.0, "completions/min_terminated_length": 158.0, "entropy": 1.1917564272880554, "epoch": 0.0016444444444444445, "frac_reward_zero_std": 0.0, "grad_norm": 1.6423976929527928, "kl": 0.006568885612068698, "learning_rate": 8.964794509221507e-08, "loss": 0.2395, "num_tokens": 378610.0, "reward": 0.75, "reward_std": 0.4629100561141968, "rewards/equation_reward_func/mean": 0.0, "rewards/equation_reward_func/std": 0.0, "rewards/format_reward_func/mean": 0.75, "rewards/format_reward_func/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 1.3353188037872314, "sampling/importance_sampling_ratio/mean": 0.6343062818050385, "sampling/importance_sampling_ratio/min": 0.06923050060868263, "sampling/sampling_logp_difference/max": 0.5188714563846588, "sampling/sampling_logp_difference/mean": 0.02377952355891466, "step": 74, "step_time": 8.72973417036701 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 881.0, "completions/max_terminated_length": 881.0, "completions/mean_length": 477.1875, "completions/mean_terminated_length": 477.1875, "completions/min_length": 173.5, "completions/min_terminated_length": 173.5, "entropy": 1.1763490214943886, "epoch": 0.0016888888888888889, "frac_reward_zero_std": 0.0, "grad_norm": 2.692320484588319, "kl": 0.007726169380475767, "learning_rate": 7.756905568047392e-08, "loss": -0.0879, "num_tokens": 388717.0, "reward": 1.0, "reward_std": 0.3535533845424652, "rewards/equation_reward_func/mean": 0.125, "rewards/equation_reward_func/std": 0.2314550280570984, "rewards/format_reward_func/mean": 0.875, "rewards/format_reward_func/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.2693214416503906, "sampling/importance_sampling_ratio/mean": 0.43672314286231995, "sampling/importance_sampling_ratio/min": 0.046458715572953224, "sampling/sampling_logp_difference/max": 0.3335939645767212, "sampling/sampling_logp_difference/mean": 0.023782381787896156, "step": 76, "step_time": 8.931666341493838 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1875, "completions/max_length": 1024.0, "completions/max_terminated_length": 626.0, "completions/mean_length": 556.125, "completions/mean_terminated_length": 446.22621154785156, "completions/min_length": 282.0, "completions/min_terminated_length": 282.0, "entropy": 1.2554637528955936, "epoch": 0.0017333333333333333, "frac_reward_zero_std": 0.0, "grad_norm": 2.2327740462000585, "kl": 0.0065791915549198166, "learning_rate": 6.621340157319996e-08, "loss": -0.0658, "num_tokens": 400055.0, "reward": 0.75, "reward_std": 0.4629100561141968, "rewards/equation_reward_func/mean": 0.0, "rewards/equation_reward_func/std": 0.0, "rewards/format_reward_func/mean": 0.75, "rewards/format_reward_func/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 1.4898787140846252, "sampling/importance_sampling_ratio/mean": 0.6342976093292236, "sampling/importance_sampling_ratio/min": 0.03369775041937828, "sampling/sampling_logp_difference/max": 0.45903223752975464, "sampling/sampling_logp_difference/mean": 0.024691437371075153, "step": 78, "step_time": 9.230151811032556 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1012.5, "completions/max_terminated_length": 1012.5, "completions/mean_length": 508.3125, "completions/mean_terminated_length": 508.3125, "completions/min_length": 198.5, "completions/min_terminated_length": 198.5, "entropy": 1.2609276585280895, "epoch": 0.0017777777777777779, "frac_reward_zero_std": 0.0, "grad_norm": 2.13225387600314, "kl": 0.007441339563229121, "learning_rate": 5.5628612330087724e-08, "loss": -0.1382, "num_tokens": 410652.0, "reward": 0.6875, "reward_std": 0.49022960662841797, "rewards/equation_reward_func/mean": 0.0, "rewards/equation_reward_func/std": 0.0, "rewards/format_reward_func/mean": 0.6875, "rewards/format_reward_func/std": 0.49022963643074036, "sampling/importance_sampling_ratio/max": 2.2199747562408447, "sampling/importance_sampling_ratio/mean": 0.8080930113792419, "sampling/importance_sampling_ratio/min": 0.17915555834770203, "sampling/sampling_logp_difference/max": 0.43730688095092773, "sampling/sampling_logp_difference/mean": 0.02470558974891901, "step": 80, "step_time": 9.341547849937342 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 705.0, "completions/max_terminated_length": 705.0, "completions/mean_length": 431.9375, "completions/mean_terminated_length": 431.9375, "completions/min_length": 275.5, "completions/min_terminated_length": 275.5, "entropy": 1.1785611994564533, "epoch": 0.0018222222222222223, "frac_reward_zero_std": 0.0, "grad_norm": 3.7533872596303937, "kl": 0.0060285410727374256, "learning_rate": 4.5859084235697235e-08, "loss": -0.0697, "num_tokens": 419979.0, "reward": 1.0, "reward_std": 0.6408699750900269, "rewards/equation_reward_func/mean": 0.1875, "rewards/equation_reward_func/std": 0.408231720328331, "rewards/format_reward_func/mean": 0.8125, "rewards/format_reward_func/std": 0.408231720328331, "sampling/importance_sampling_ratio/max": 2.7441056966781616, "sampling/importance_sampling_ratio/mean": 0.8698084950447083, "sampling/importance_sampling_ratio/min": 0.05789299122989178, "sampling/sampling_logp_difference/max": 0.9367153644561768, "sampling/sampling_logp_difference/mean": 0.024671697057783604, "step": 82, "step_time": 8.355979559826665 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 905.0, "completions/max_terminated_length": 905.0, "completions/mean_length": 495.5, "completions/mean_terminated_length": 495.5, "completions/min_length": 230.5, "completions/min_terminated_length": 230.5, "entropy": 1.2624935917556286, "epoch": 0.0018666666666666666, "frac_reward_zero_std": 0.0, "grad_norm": 1.7840327619724896, "kl": 0.006831162696471438, "learning_rate": 3.6945794086007705e-08, "loss": -0.0282, "num_tokens": 430347.0, "reward": 0.8125, "reward_std": 0.408231720328331, "rewards/equation_reward_func/mean": 0.0, "rewards/equation_reward_func/std": 0.0, "rewards/format_reward_func/mean": 0.8125, "rewards/format_reward_func/std": 0.408231720328331, "sampling/importance_sampling_ratio/max": 1.5519986152648926, "sampling/importance_sampling_ratio/mean": 0.5716961026191711, "sampling/importance_sampling_ratio/min": 0.15530326962471008, "sampling/sampling_logp_difference/max": 0.5926408171653748, "sampling/sampling_logp_difference/mean": 0.024989251978695393, "step": 84, "step_time": 8.7652991093928 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 778.5, "completions/max_terminated_length": 739.0, "completions/mean_length": 491.0, "completions/mean_terminated_length": 459.4821472167969, "completions/min_length": 261.5, "completions/min_terminated_length": 261.5, "entropy": 1.1592534072697163, "epoch": 0.001911111111111111, "frac_reward_zero_std": 0.0, "grad_norm": 3.889869064786408, "kl": 0.008921730826841667, "learning_rate": 2.892612731749414e-08, "loss": -0.0768, "num_tokens": 440675.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/equation_reward_func/mean": 0.0, "rewards/equation_reward_func/std": 0.0, "rewards/format_reward_func/mean": 0.875, "rewards/format_reward_func/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.261430025100708, "sampling/importance_sampling_ratio/mean": 0.5434921085834503, "sampling/importance_sampling_ratio/min": 0.12478071637451649, "sampling/sampling_logp_difference/max": 1.3635588884353638, "sampling/sampling_logp_difference/mean": 0.024433997459709644, "step": 86, "step_time": 8.525219976552762 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 683.0, "completions/max_terminated_length": 683.0, "completions/mean_length": 363.5, "completions/mean_terminated_length": 363.5, "completions/min_length": 165.5, "completions/min_terminated_length": 165.5, "entropy": 1.1377994492650032, "epoch": 0.0019555555555555554, "frac_reward_zero_std": 0.0, "grad_norm": 6.134372721816042, "kl": 0.015324803374824114, "learning_rate": 2.183372119961499e-08, "loss": 0.5464, "num_tokens": 448899.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/equation_reward_func/mean": 0.0, "rewards/equation_reward_func/std": 0.0, "rewards/format_reward_func/mean": 0.875, "rewards/format_reward_func/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 2.269628405570984, "sampling/importance_sampling_ratio/mean": 1.1133230924606323, "sampling/importance_sampling_ratio/min": 0.2772098332643509, "sampling/sampling_logp_difference/max": 0.8273682594299316, "sampling/sampling_logp_difference/mean": 0.02418291289359331, "step": 88, "step_time": 8.307329173432663 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 830.0, "completions/max_terminated_length": 830.0, "completions/mean_length": 560.9375, "completions/mean_terminated_length": 560.9375, "completions/min_length": 261.0, "completions/min_terminated_length": 261.0, "entropy": 1.2554445713758469, "epoch": 0.002, "frac_reward_zero_std": 0.0, "grad_norm": 2.8357390700930227, "kl": 0.015233526355586946, "learning_rate": 1.5698323748414122e-08, "loss": -0.2772, "num_tokens": 460354.0, "reward": 0.8125, "reward_std": 0.5260358154773712, "rewards/equation_reward_func/mean": 0.0625, "rewards/equation_reward_func/std": 0.1767766922712326, "rewards/format_reward_func/mean": 0.75, "rewards/format_reward_func/std": 0.4355513006448746, "sampling/importance_sampling_ratio/max": 2.431339144706726, "sampling/importance_sampling_ratio/mean": 0.6834549903869629, "sampling/importance_sampling_ratio/min": 0.01039754506200552, "sampling/sampling_logp_difference/max": 0.5135123133659363, "sampling/sampling_logp_difference/mean": 0.02495667804032564, "step": 90, "step_time": 8.614867440075614 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 924.0, "completions/max_terminated_length": 772.5, "completions/mean_length": 546.5, "completions/mean_terminated_length": 516.4196472167969, "completions/min_length": 266.5, "completions/min_terminated_length": 266.5, "entropy": 1.4302296712994576, "epoch": 0.0020444444444444447, "frac_reward_zero_std": 0.0, "grad_norm": 2.4766318095819377, "kl": 0.006567903692484833, "learning_rate": 1.054566895300324e-08, "loss": -0.1519, "num_tokens": 471546.0, "reward": 0.5625, "reward_std": 0.49022960662841797, "rewards/equation_reward_func/mean": 0.0, "rewards/equation_reward_func/std": 0.0, "rewards/format_reward_func/mean": 0.5625, "rewards/format_reward_func/std": 0.49022963643074036, "sampling/importance_sampling_ratio/max": 2.179307520389557, "sampling/importance_sampling_ratio/mean": 0.7372348606586456, "sampling/importance_sampling_ratio/min": 0.0416359007358551, "sampling/sampling_logp_difference/max": 0.8628695011138916, "sampling/sampling_logp_difference/mean": 0.025372425094246864, "step": 92, "step_time": 9.102577322046272 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 753.0, "completions/max_terminated_length": 753.0, "completions/mean_length": 468.8125, "completions/mean_terminated_length": 468.8125, "completions/min_length": 178.5, "completions/min_terminated_length": 178.5, "entropy": 1.2605133689939976, "epoch": 0.002088888888888889, "frac_reward_zero_std": 0.0, "grad_norm": 1.5504326025550264, "kl": 0.00589993035828229, "learning_rate": 6.397368838268496e-09, "loss": -0.1529, "num_tokens": 481495.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/equation_reward_func/mean": 0.0, "rewards/equation_reward_func/std": 0.0, "rewards/format_reward_func/mean": 0.875, "rewards/format_reward_func/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 2.2573115825653076, "sampling/importance_sampling_ratio/mean": 0.8763566613197327, "sampling/importance_sampling_ratio/min": 0.26127955317497253, "sampling/sampling_logp_difference/max": 0.3228236436843872, "sampling/sampling_logp_difference/mean": 0.02433642279356718, "step": 94, "step_time": 8.598844347870909 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 454.0, "completions/max_terminated_length": 454.0, "completions/mean_length": 321.9375, "completions/mean_terminated_length": 321.9375, "completions/min_length": 197.0, "completions/min_terminated_length": 197.0, "entropy": 1.196368370205164, "epoch": 0.0021333333333333334, "frac_reward_zero_std": 0.5, "grad_norm": 0.004081368931451629, "kl": 0.013201179972384125, "learning_rate": 3.2708228165273244e-09, "loss": -0.0133, "num_tokens": 489038.0, "reward": 1.0625, "reward_std": 0.3204349875450134, "rewards/equation_reward_func/mean": 0.125, "rewards/equation_reward_func/std": 0.2314550280570984, "rewards/format_reward_func/mean": 0.9375, "rewards/format_reward_func/std": 0.1767766922712326, "sampling/importance_sampling_ratio/max": 1.5249131321907043, "sampling/importance_sampling_ratio/mean": 0.6771150529384613, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 0.3968193531036377, "sampling/sampling_logp_difference/mean": 0.02468986716121435, "step": 96, "step_time": 7.818771839141846 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 542.0, "completions/max_terminated_length": 542.0, "completions/mean_length": 361.875, "completions/mean_terminated_length": 361.875, "completions/min_length": 201.0, "completions/min_terminated_length": 201.0, "entropy": 1.1868297532200813, "epoch": 0.0021777777777777776, "frac_reward_zero_std": 0.0, "grad_norm": 2.254444002886367, "kl": 0.011681211748509668, "learning_rate": 1.1791447083465133e-09, "loss": 0.4881, "num_tokens": 497244.0, "reward": 1.125, "reward_std": 0.49721167981624603, "rewards/equation_reward_func/mean": 0.1875, "rewards/equation_reward_func/std": 0.408231720328331, "rewards/format_reward_func/mean": 0.9375, "rewards/format_reward_func/std": 0.1767766922712326, "sampling/importance_sampling_ratio/max": 2.307553768157959, "sampling/importance_sampling_ratio/mean": 0.7432051002979279, "sampling/importance_sampling_ratio/min": 0.06528420001268387, "sampling/sampling_logp_difference/max": 0.4255487322807312, "sampling/sampling_logp_difference/mean": 0.025311018340289593, "step": 98, "step_time": 7.8635854490567 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 583.0, "completions/max_terminated_length": 583.0, "completions/mean_length": 371.6875, "completions/mean_terminated_length": 371.6875, "completions/min_length": 173.5, "completions/min_terminated_length": 173.5, "entropy": 1.1904011890292168, "epoch": 0.0022222222222222222, "frac_reward_zero_std": 0.0, "grad_norm": 6.048771291369092, "kl": 0.0074510836420813575, "learning_rate": 1.3110773862126667e-10, "loss": 0.5377, "num_tokens": 505607.0, "reward": 1.0, "reward_std": 0.49721167981624603, "rewards/equation_reward_func/mean": 0.125, "rewards/equation_reward_func/std": 0.2314550280570984, "rewards/format_reward_func/mean": 0.875, "rewards/format_reward_func/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.8672412633895874, "sampling/importance_sampling_ratio/mean": 0.7961099743843079, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 0.4504559636116028, "sampling/sampling_logp_difference/mean": 0.024699319154024124, "step": 100, "step_time": 8.325517300749198 }, { "epoch": 0.0022222222222222222, "step": 100, "total_flos": 0.0, "train_loss": 0.003586103413254023, "train_runtime": 1244.4861, "train_samples_per_second": 0.643, "train_steps_per_second": 0.08 } ], "logging_steps": 2, "max_steps": 100, "num_input_tokens_seen": 505607, "num_train_epochs": 1, "save_steps": 25, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": true, "should_training_stop": true }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 1, "trial_name": null, "trial_params": null }