{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 0.12002133712660028, "eval_steps": 500, "global_step": 450, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0185546875, "completions/max_length": 1024.0, "completions/max_terminated_length": 941.5, "completions/mean_length": 228.861328125, "completions/mean_terminated_length": 213.82315063476562, "completions/min_length": 14.5, "completions/min_terminated_length": 14.5, "entropy": 0.9906870043940015, "epoch": 0.0005334281650071124, "frac_reward_zero_std": 0.8515625, "grad_norm": 0.16291454434394836, "kl": 0.0, "learning_rate": 3.571428571428571e-08, "loss": 0.0119, "num_tokens": 390482.0, "reward": 0.021484375, "reward_std": 0.05504363030195236, "rewards/equation_reward_func/mean": 0.001953125, "rewards/equation_reward_func/std": 0.03121940791606903, "rewards/format_reward_func/mean": 0.01953125, "rewards/format_reward_func/std": 0.13361797481775284, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9999333918094635, "sampling/importance_sampling_ratio/min": 0.44458194077014923, "sampling/sampling_logp_difference/max": 1.0505925416946411, "sampling/sampling_logp_difference/mean": 0.02098759263753891, "step": 2 }, { "clip_ratio/high_max": 5.300016507842681e-05, "clip_ratio/high_mean": 5.300016507842681e-05, "clip_ratio/low_mean": 0.00010970261913219777, "clip_ratio/low_min": 0.00010970261913219777, "clip_ratio/region_mean": 0.00016270278421062458, "completions/clipped_ratio": 0.02734375, "completions/max_length": 1024.0, "completions/max_terminated_length": 966.0, "completions/mean_length": 248.947265625, "completions/mean_terminated_length": 227.15863037109375, "completions/min_length": 20.0, "completions/min_terminated_length": 20.0, "entropy": 0.9930441313319736, "epoch": 0.0010668563300142249, "frac_reward_zero_std": 0.8125, "grad_norm": 0.2261732518672943, "kl": 0.0007615485147754145, "learning_rate": 1.0714285714285713e-07, "loss": 0.0266, "num_tokens": 596199.0, "reward": 0.0234375, "reward_std": 0.06629125773906708, "rewards/equation_reward_func/mean": 0.0, "rewards/equation_reward_func/std": 0.0, "rewards/format_reward_func/mean": 0.0234375, "rewards/format_reward_func/std": 0.15143637359142303, "sampling/importance_sampling_ratio/max": 1.7380328178405762, "sampling/importance_sampling_ratio/mean": 0.999877393245697, "sampling/importance_sampling_ratio/min": 0.4712234437465668, "sampling/sampling_logp_difference/max": 0.7524229288101196, "sampling/sampling_logp_difference/mean": 0.020538652315735817, "step": 4 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0001006675087814478, "clip_ratio/low_min": 0.0001006675087814478, "clip_ratio/region_mean": 0.0001006675087814478, "completions/clipped_ratio": 0.021484375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1005.0, "completions/mean_length": 243.634765625, "completions/mean_terminated_length": 226.50099182128906, "completions/min_length": 19.0, "completions/min_terminated_length": 19.0, "entropy": 0.9561915049950281, "epoch": 0.001600284495021337, "frac_reward_zero_std": 0.828125, "grad_norm": 0.19601207971572876, "kl": 0.0007531659551508104, "learning_rate": 1.7857142857142858e-07, "loss": 0.0277, "num_tokens": 798924.0, "reward": 0.03125, "reward_std": 0.0684555172920227, "rewards/equation_reward_func/mean": 0.001953125, "rewards/equation_reward_func/std": 0.04419417306780815, "rewards/format_reward_func/mean": 0.029296875, "rewards/format_reward_func/std": 0.16880230605602264, "sampling/importance_sampling_ratio/max": 1.6818383932113647, "sampling/importance_sampling_ratio/mean": 0.9998596906661987, "sampling/importance_sampling_ratio/min": 0.5696869492530823, "sampling/sampling_logp_difference/max": 0.5626683235168457, "sampling/sampling_logp_difference/mean": 0.02069748565554619, "step": 6 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 3.0059856928548674e-05, "clip_ratio/low_min": 3.0059856928548674e-05, "clip_ratio/region_mean": 3.0059856928548674e-05, "completions/clipped_ratio": 0.01953125, "completions/max_length": 1024.0, "completions/max_terminated_length": 1003.0, "completions/mean_length": 228.125, "completions/mean_terminated_length": 212.2709197998047, "completions/min_length": 16.0, "completions/min_terminated_length": 16.0, "entropy": 0.9969107690784667, "epoch": 0.0021337126600284497, "frac_reward_zero_std": 0.921875, "grad_norm": 0.17554284632205963, "kl": 0.0007573424453666019, "learning_rate": 2.5e-07, "loss": 0.0153, "num_tokens": 993724.0, "reward": 0.009765625, "reward_std": 0.027621358633041382, "rewards/equation_reward_func/mean": 0.0, "rewards/equation_reward_func/std": 0.0, "rewards/format_reward_func/mean": 0.009765625, "rewards/format_reward_func/std": 0.09843364357948303, "sampling/importance_sampling_ratio/max": 1.7989540100097656, "sampling/importance_sampling_ratio/mean": 0.992868185043335, "sampling/importance_sampling_ratio/min": 3.239048959602364e-11, "sampling/sampling_logp_difference/max": 24.153156280517578, "sampling/sampling_logp_difference/mean": 0.05915800854563713, "step": 8 }, { "clip_ratio/high_max": 2.6382262553347067e-05, "clip_ratio/high_mean": 2.6382262553347067e-05, "clip_ratio/low_mean": 6.268165695170562e-05, "clip_ratio/low_min": 6.268165695170562e-05, "clip_ratio/region_mean": 8.906391950505268e-05, "completions/clipped_ratio": 0.025390625, "completions/max_length": 1024.0, "completions/max_terminated_length": 985.0, "completions/mean_length": 236.822265625, "completions/mean_terminated_length": 216.31463623046875, "completions/min_length": 12.0, "completions/min_terminated_length": 12.0, "entropy": 1.0309493674172296, "epoch": 0.0026671408250355617, "frac_reward_zero_std": 0.828125, "grad_norm": 0.2110099196434021, "kl": 0.000838718113502384, "learning_rate": 3.2142857142857145e-07, "loss": 0.0136, "num_tokens": 1193225.0, "reward": 0.0234375, "reward_std": 0.06247568875551224, "rewards/equation_reward_func/mean": 0.0, "rewards/equation_reward_func/std": 0.0, "rewards/format_reward_func/mean": 0.0234375, "rewards/format_reward_func/std": 0.15143637359142303, "sampling/importance_sampling_ratio/max": 1.8266597986221313, "sampling/importance_sampling_ratio/mean": 1.0000933408737183, "sampling/importance_sampling_ratio/min": 0.4933336079120636, "sampling/sampling_logp_difference/max": 0.7065696716308594, "sampling/sampling_logp_difference/mean": 0.020812492817640305, "step": 10 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0001034049900125764, "clip_ratio/low_min": 0.0001034049900125764, "clip_ratio/region_mean": 0.0001034049900125764, "completions/clipped_ratio": 0.03125, "completions/max_length": 1024.0, "completions/max_terminated_length": 987.0, "completions/mean_length": 230.103515625, "completions/mean_terminated_length": 204.4939422607422, "completions/min_length": 13.0, "completions/min_terminated_length": 13.0, "entropy": 1.0229314300749037, "epoch": 0.003200568990042674, "frac_reward_zero_std": 0.890625, "grad_norm": 0.1713971197605133, "kl": 0.0007667660134352951, "learning_rate": 3.928571428571428e-07, "loss": 0.0199, "num_tokens": 1389166.0, "reward": 0.015625, "reward_std": 0.04037860035896301, "rewards/equation_reward_func/mean": 0.0, "rewards/equation_reward_func/std": 0.0, "rewards/format_reward_func/mean": 0.015625, "rewards/format_reward_func/std": 0.12414088100194931, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9999401569366455, "sampling/importance_sampling_ratio/min": 0.4219718873500824, "sampling/sampling_logp_difference/max": 0.9450585842132568, "sampling/sampling_logp_difference/mean": 0.02105509303510189, "step": 12 }, { "clip_ratio/high_max": 8.771006832830608e-06, "clip_ratio/high_mean": 8.771006832830608e-06, "clip_ratio/low_mean": 9.435887235061576e-05, "clip_ratio/low_min": 9.435887235061576e-05, "clip_ratio/region_mean": 0.00010312987918344636, "completions/clipped_ratio": 0.033203125, "completions/max_length": 1024.0, "completions/max_terminated_length": 998.0, "completions/mean_length": 237.861328125, "completions/mean_terminated_length": 210.86264038085938, "completions/min_length": 15.0, "completions/min_terminated_length": 15.0, "entropy": 0.9931543800565932, "epoch": 0.0037339971550497866, "frac_reward_zero_std": 0.84375, "grad_norm": 0.18590842187404633, "kl": 0.0008038054689273445, "learning_rate": 4.6428571428571427e-07, "loss": 0.0143, "num_tokens": 1589071.0, "reward": 0.01953125, "reward_std": 0.055242717266082764, "rewards/equation_reward_func/mean": 0.001953125, "rewards/equation_reward_func/std": 0.04419417306780815, "rewards/format_reward_func/mean": 0.017578125, "rewards/format_reward_func/std": 0.13154059648513794, "sampling/importance_sampling_ratio/max": 1.8855645656585693, "sampling/importance_sampling_ratio/mean": 0.9999428391456604, "sampling/importance_sampling_ratio/min": 0.2005501091480255, "sampling/sampling_logp_difference/max": 1.6066911220550537, "sampling/sampling_logp_difference/mean": 0.020327817648649216, "step": 14 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 9.463882532953803e-05, "clip_ratio/low_min": 9.463882532953803e-05, "clip_ratio/region_mean": 9.463882532953803e-05, "completions/clipped_ratio": 0.029296875, "completions/max_length": 1024.0, "completions/max_terminated_length": 984.0, "completions/mean_length": 250.32421875, "completions/mean_terminated_length": 226.9738311767578, "completions/min_length": 14.0, "completions/min_terminated_length": 14.0, "entropy": 0.9750576963027319, "epoch": 0.004267425320056899, "frac_reward_zero_std": 0.828125, "grad_norm": 0.19739331305027008, "kl": 0.0010235626525374958, "learning_rate": 4.999935101463869e-07, "loss": 0.0143, "num_tokens": 1795301.0, "reward": 0.0234375, "reward_std": 0.06247568875551224, "rewards/equation_reward_func/mean": 0.0, "rewards/equation_reward_func/std": 0.0, "rewards/format_reward_func/mean": 0.0234375, "rewards/format_reward_func/std": 0.15143637359142303, "sampling/importance_sampling_ratio/max": 1.8375256061553955, "sampling/importance_sampling_ratio/mean": 0.9952285885810852, "sampling/importance_sampling_ratio/min": 7.352532471749385e-11, "sampling/sampling_logp_difference/max": 23.333391189575195, "sampling/sampling_logp_difference/mean": 0.07141915708780289, "step": 16 }, { "clip_ratio/high_max": 7.348618560677602e-06, "clip_ratio/high_mean": 7.348618560677602e-06, "clip_ratio/low_mean": 6.096324634401956e-05, "clip_ratio/low_min": 6.096324634401956e-05, "clip_ratio/region_mean": 6.831186490469716e-05, "completions/clipped_ratio": 0.0224609375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1007.0, "completions/mean_length": 231.904296875, "completions/mean_terminated_length": 213.72539520263672, "completions/min_length": 15.0, "completions/min_terminated_length": 15.0, "entropy": 1.015358070532481, "epoch": 0.004800853485064011, "frac_reward_zero_std": 0.6953125, "grad_norm": 0.31367021799087524, "kl": 0.0011365925408123683, "learning_rate": 4.999415933391384e-07, "loss": 0.0309, "num_tokens": 2188979.0, "reward": 0.0458984375, "reward_std": 0.11455809697508812, "rewards/equation_reward_func/mean": 0.0029296875, "rewards/equation_reward_func/std": 0.053316494449973106, "rewards/format_reward_func/mean": 0.04296875, "rewards/format_reward_func/std": 0.20246700942516327, "sampling/importance_sampling_ratio/max": 1.9270648956298828, "sampling/importance_sampling_ratio/mean": 1.000028133392334, "sampling/importance_sampling_ratio/min": 0.23970969766378403, "sampling/sampling_logp_difference/max": 1.4344506859779358, "sampling/sampling_logp_difference/mean": 0.02102941181510687, "step": 18 }, { "clip_ratio/high_max": 8.455857298233443e-05, "clip_ratio/high_mean": 8.455857298233443e-05, "clip_ratio/low_mean": 0.00023414362092605897, "clip_ratio/low_min": 0.00023414362092605897, "clip_ratio/region_mean": 0.0003187021922915139, "completions/clipped_ratio": 0.009765625, "completions/max_length": 1024.0, "completions/max_terminated_length": 928.0, "completions/mean_length": 208.12109375, "completions/mean_terminated_length": 200.074951171875, "completions/min_length": 14.0, "completions/min_terminated_length": 14.0, "entropy": 0.9739413443538878, "epoch": 0.005334281650071123, "frac_reward_zero_std": 0.71875, "grad_norm": 0.29864609241485596, "kl": 0.0018909820646513253, "learning_rate": 4.998377705063407e-07, "loss": 0.043, "num_tokens": 2373873.0, "reward": 0.05859375, "reward_std": 0.1165214255452156, "rewards/equation_reward_func/mean": 0.0, "rewards/equation_reward_func/std": 0.0, "rewards/format_reward_func/mean": 0.05859375, "rewards/format_reward_func/std": 0.23509246110916138, "sampling/importance_sampling_ratio/max": 1.7688930034637451, "sampling/importance_sampling_ratio/mean": 0.9998641610145569, "sampling/importance_sampling_ratio/min": 0.5252137184143066, "sampling/sampling_logp_difference/max": 0.6439499855041504, "sampling/sampling_logp_difference/mean": 0.020961318165063858, "step": 20 }, { "clip_ratio/high_max": 8.217327882044224e-05, "clip_ratio/high_mean": 8.217327882044224e-05, "clip_ratio/low_mean": 0.00020888712970190682, "clip_ratio/low_min": 0.00020888712970190682, "clip_ratio/region_mean": 0.00029106041256454773, "completions/clipped_ratio": 0.017578125, "completions/max_length": 1024.0, "completions/max_terminated_length": 1006.0, "completions/mean_length": 217.525390625, "completions/mean_terminated_length": 203.0954132080078, "completions/min_length": 13.0, "completions/min_terminated_length": 13.0, "entropy": 0.9706550488869349, "epoch": 0.005867709815078236, "frac_reward_zero_std": 0.515625, "grad_norm": 0.32972589135169983, "kl": 0.0031107893123084474, "learning_rate": 4.996820632091536e-07, "loss": 0.0343, "num_tokens": 2563390.0, "reward": 0.078125, "reward_std": 0.1843322515487671, "rewards/equation_reward_func/mean": 0.00390625, "rewards/equation_reward_func/std": 0.06243881583213806, "rewards/format_reward_func/mean": 0.07421875, "rewards/format_reward_func/std": 0.2623828947544098, "sampling/importance_sampling_ratio/max": 1.894946575164795, "sampling/importance_sampling_ratio/mean": 0.9999268054962158, "sampling/importance_sampling_ratio/min": 0.11174897849559784, "sampling/sampling_logp_difference/max": 2.191500186920166, "sampling/sampling_logp_difference/mean": 0.021253950893878937, "step": 22 }, { "clip_ratio/high_max": 3.2112408209488625e-05, "clip_ratio/high_mean": 3.2112408209488625e-05, "clip_ratio/low_mean": 0.00027551787813232903, "clip_ratio/low_min": 0.00027551787813232903, "clip_ratio/region_mean": 0.00030763028634181764, "completions/clipped_ratio": 0.0234375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.0, "completions/mean_length": 194.970703125, "completions/mean_terminated_length": 175.07400512695312, "completions/min_length": 14.0, "completions/min_terminated_length": 14.0, "entropy": 0.9849063654740652, "epoch": 0.006401137980085348, "frac_reward_zero_std": 0.328125, "grad_norm": 0.6001269817352295, "kl": 0.003953174649116893, "learning_rate": 4.994745037837194e-07, "loss": 0.0869, "num_tokens": 2741335.0, "reward": 0.126953125, "reward_std": 0.2717152237892151, "rewards/equation_reward_func/mean": 0.0, "rewards/equation_reward_func/std": 0.0, "rewards/format_reward_func/mean": 0.126953125, "rewards/format_reward_func/std": 0.33324605226516724, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9998724460601807, "sampling/importance_sampling_ratio/min": 0.4356817901134491, "sampling/sampling_logp_difference/max": 0.8308432102203369, "sampling/sampling_logp_difference/mean": 0.02183392085134983, "step": 24 }, { "clip_ratio/high_max": 0.0003703655949923106, "clip_ratio/high_mean": 0.0003703655949923106, "clip_ratio/low_mean": 0.0029490124684849028, "clip_ratio/low_min": 0.0029490124684849028, "clip_ratio/region_mean": 0.0033193781152173565, "completions/clipped_ratio": 0.013671875, "completions/max_length": 1024.0, "completions/max_terminated_length": 983.0, "completions/mean_length": 180.943359375, "completions/mean_terminated_length": 169.25743103027344, "completions/min_length": 11.0, "completions/min_terminated_length": 11.0, "entropy": 1.019554615020752, "epoch": 0.006934566145092461, "frac_reward_zero_std": 0.21875, "grad_norm": 0.8571369051933289, "kl": 0.010997309517632756, "learning_rate": 4.992151353344481e-07, "loss": 0.0869, "num_tokens": 2912138.0, "reward": 0.171875, "reward_std": 0.3274696171283722, "rewards/equation_reward_func/mean": 0.001953125, "rewards/equation_reward_func/std": 0.04419417306780815, "rewards/format_reward_func/mean": 0.169921875, "rewards/format_reward_func/std": 0.3759314715862274, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9895955324172974, "sampling/importance_sampling_ratio/min": 5.196759342296353e-11, "sampling/sampling_logp_difference/max": 23.680400848388672, "sampling/sampling_logp_difference/mean": 0.11238687485456467, "step": 26 }, { "clip_ratio/high_max": 0.0007882783518450904, "clip_ratio/high_mean": 0.0007882783518450904, "clip_ratio/low_mean": 0.00845093373532614, "clip_ratio/low_min": 0.00845093373532614, "clip_ratio/region_mean": 0.009239212098893605, "completions/clipped_ratio": 0.0078125, "completions/max_length": 1024.0, "completions/max_terminated_length": 1001.0, "completions/mean_length": 156.486328125, "completions/mean_terminated_length": 149.655517578125, "completions/min_length": 14.0, "completions/min_terminated_length": 14.0, "entropy": 0.9848703841368357, "epoch": 0.007467994310099573, "frac_reward_zero_std": 0.078125, "grad_norm": 0.9129703640937805, "kl": 0.0329602628480643, "learning_rate": 4.989040117250646e-07, "loss": 0.1433, "num_tokens": 3070395.0, "reward": 0.279296875, "reward_std": 0.42425647377967834, "rewards/equation_reward_func/mean": 0.00390625, "rewards/equation_reward_func/std": 0.06243881583213806, "rewards/format_reward_func/mean": 0.275390625, "rewards/format_reward_func/std": 0.44714778661727905, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9907291531562805, "sampling/importance_sampling_ratio/min": 3.045626945086255e-11, "sampling/sampling_logp_difference/max": 24.21472930908203, "sampling/sampling_logp_difference/mean": 0.1076340302824974, "step": 28 }, { "clip_ratio/high_max": 0.0006181110511533916, "clip_ratio/high_mean": 0.0006181110511533916, "clip_ratio/low_mean": 0.002150450627798111, "clip_ratio/low_min": 0.002150450627798111, "clip_ratio/region_mean": 0.0027685616951202974, "completions/clipped_ratio": 0.0078125, "completions/max_length": 1024.0, "completions/max_terminated_length": 888.0, "completions/mean_length": 127.173828125, "completions/mean_terminated_length": 120.1122055053711, "completions/min_length": 18.0, "completions/min_terminated_length": 18.0, "entropy": 0.902042117383745, "epoch": 0.008001422475106686, "frac_reward_zero_std": 0.0, "grad_norm": 0.9917025566101074, "kl": 0.026186900937722787, "learning_rate": 4.985411975674243e-07, "loss": 0.1706, "num_tokens": 3213676.0, "reward": 0.458984375, "reward_std": 0.4996362030506134, "rewards/equation_reward_func/mean": 0.00390625, "rewards/equation_reward_func/std": 0.06243881583213806, "rewards/format_reward_func/mean": 0.455078125, "rewards/format_reward_func/std": 0.4984649419784546, "sampling/importance_sampling_ratio/max": 1.58962881565094, "sampling/importance_sampling_ratio/mean": 1.0000369548797607, "sampling/importance_sampling_ratio/min": 0.5563350915908813, "sampling/sampling_logp_difference/max": 0.5863845348358154, "sampling/sampling_logp_difference/mean": 0.020490314811468124, "step": 30 }, { "clip_ratio/high_max": 0.0004448202024731371, "clip_ratio/high_mean": 0.0004448202024731371, "clip_ratio/low_mean": 0.0014546795913095896, "clip_ratio/low_min": 0.0014546795913095896, "clip_ratio/region_mean": 0.0018994997986333652, "completions/clipped_ratio": 0.005859375, "completions/max_length": 1024.0, "completions/max_terminated_length": 801.0, "completions/mean_length": 117.958984375, "completions/mean_terminated_length": 112.61886596679688, "completions/min_length": 18.0, "completions/min_terminated_length": 18.0, "entropy": 0.8814441644483142, "epoch": 0.008534850640113799, "frac_reward_zero_std": 0.0, "grad_norm": 1.08668053150177, "kl": 0.033620989260574184, "learning_rate": 4.981267682080939e-07, "loss": 0.2037, "num_tokens": 3352399.0, "reward": 0.544921875, "reward_std": 0.5004862546920776, "rewards/equation_reward_func/mean": 0.00390625, "rewards/equation_reward_func/std": 0.06243881583213806, "rewards/format_reward_func/mean": 0.541015625, "rewards/format_reward_func/std": 0.49880221486091614, "sampling/importance_sampling_ratio/max": 1.592621088027954, "sampling/importance_sampling_ratio/mean": 1.0002690553665161, "sampling/importance_sampling_ratio/min": 0.627295732498169, "sampling/sampling_logp_difference/max": 0.4663372039794922, "sampling/sampling_logp_difference/mean": 0.020882319658994675, "step": 32 }, { "clip_ratio/high_max": 0.00021157415337964066, "clip_ratio/high_mean": 0.00021157415337964066, "clip_ratio/low_mean": 0.0010422569240391669, "clip_ratio/low_min": 0.0010422569240391669, "clip_ratio/region_mean": 0.0012538310580162539, "completions/clipped_ratio": 0.0029296875, "completions/max_length": 932.5, "completions/max_terminated_length": 879.0, "completions/mean_length": 97.865234375, "completions/mean_terminated_length": 95.14189910888672, "completions/min_length": 17.5, "completions/min_terminated_length": 17.5, "entropy": 0.8883479121658537, "epoch": 0.00906827880512091, "frac_reward_zero_std": 0.046875, "grad_norm": 1.2837241888046265, "kl": 0.049437474925071, "learning_rate": 4.976608097127043e-07, "loss": 0.1989, "num_tokens": 3608821.0, "reward": 0.666015625, "reward_std": 0.4548095315694809, "rewards/equation_reward_func/mean": 0.0029296875, "rewards/equation_reward_func/std": 0.053316494449973106, "rewards/format_reward_func/mean": 0.6630859375, "rewards/format_reward_func/std": 0.4725191593170166, "sampling/importance_sampling_ratio/max": 1.6358660459518433, "sampling/importance_sampling_ratio/mean": 0.9954747259616852, "sampling/importance_sampling_ratio/min": 0.2847309411318844, "sampling/sampling_logp_difference/max": 11.593867659568787, "sampling/sampling_logp_difference/mean": 0.07399317715317011, "step": 34 }, { "clip_ratio/high_max": 0.0006300570425486916, "clip_ratio/high_mean": 0.0006300570425486916, "clip_ratio/low_mean": 0.004105484530252094, "clip_ratio/low_min": 0.004105484530252094, "clip_ratio/region_mean": 0.004735541625349369, "completions/clipped_ratio": 0.005859375, "completions/max_length": 1024.0, "completions/max_terminated_length": 845.0, "completions/mean_length": 95.708984375, "completions/mean_terminated_length": 90.23772430419922, "completions/min_length": 25.0, "completions/min_terminated_length": 25.0, "entropy": 0.8173519985543357, "epoch": 0.009601706970128023, "frac_reward_zero_std": 0.046875, "grad_norm": 1.169929027557373, "kl": 0.07463782078896959, "learning_rate": 4.97143418848077e-07, "loss": 0.1797, "num_tokens": 3736000.0, "reward": 0.75, "reward_std": 0.4411673843860626, "rewards/equation_reward_func/mean": 0.005859375, "rewards/equation_reward_func/std": 0.07639661431312561, "rewards/format_reward_func/mean": 0.744140625, "rewards/format_reward_func/std": 0.43676990270614624, "sampling/importance_sampling_ratio/max": 1.7114160060882568, "sampling/importance_sampling_ratio/mean": 0.9796027541160583, "sampling/importance_sampling_ratio/min": 5.638221139969701e-11, "sampling/sampling_logp_difference/max": 23.598867416381836, "sampling/sampling_logp_difference/mean": 0.26203441619873047, "step": 36 }, { "clip_ratio/high_max": 0.0004559664189906067, "clip_ratio/high_mean": 0.0004559664189906067, "clip_ratio/low_mean": 0.00839672572813773, "clip_ratio/low_min": 0.00839672572813773, "clip_ratio/region_mean": 0.008852692146319896, "completions/clipped_ratio": 0.001953125, "completions/max_length": 1024.0, "completions/max_terminated_length": 706.0, "completions/mean_length": 75.271484375, "completions/mean_terminated_length": 73.41487121582031, "completions/min_length": 25.0, "completions/min_terminated_length": 25.0, "entropy": 0.8584042108721204, "epoch": 0.010135135135135136, "frac_reward_zero_std": 0.28125, "grad_norm": 1.4801753759384155, "kl": 28.380483800234895, "learning_rate": 4.965747030621286e-07, "loss": 0.2329, "num_tokens": 3852635.0, "reward": 0.865234375, "reward_std": 0.2919681668281555, "rewards/equation_reward_func/mean": 0.00390625, "rewards/equation_reward_func/std": 0.06243881583213806, "rewards/format_reward_func/mean": 0.861328125, "rewards/format_reward_func/std": 0.34594178199768066, "sampling/importance_sampling_ratio/max": 1.733668327331543, "sampling/importance_sampling_ratio/mean": 1.0000662803649902, "sampling/importance_sampling_ratio/min": 0.4352138638496399, "sampling/sampling_logp_difference/max": 0.8319177627563477, "sampling/sampling_logp_difference/mean": 0.01950383372604847, "step": 38 }, { "clip_ratio/high_max": 0.0004300644068280235, "clip_ratio/high_mean": 0.0004300644068280235, "clip_ratio/low_mean": 0.0008942384576787137, "clip_ratio/low_min": 0.0008942384576787137, "clip_ratio/region_mean": 0.0013243028612729784, "completions/clipped_ratio": 0.00390625, "completions/max_length": 1024.0, "completions/max_terminated_length": 604.0, "completions/mean_length": 74.28515625, "completions/mean_terminated_length": 70.560791015625, "completions/min_length": 21.0, "completions/min_terminated_length": 21.0, "entropy": 0.828421617547671, "epoch": 0.010668563300142247, "frac_reward_zero_std": 0.5, "grad_norm": 266.15692138671875, "kl": 10.770809243329698, "learning_rate": 4.959547804615562e-07, "loss": 0.1596, "num_tokens": 3968701.0, "reward": 0.91015625, "reward_std": 0.2011794149875641, "rewards/equation_reward_func/mean": 0.001953125, "rewards/equation_reward_func/std": 0.04419417306780815, "rewards/format_reward_func/mean": 0.908203125, "rewards/format_reward_func/std": 0.289021372795105, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9749342203140259, "sampling/importance_sampling_ratio/min": 7.815229713381555e-11, "sampling/sampling_logp_difference/max": 23.272361755371094, "sampling/sampling_logp_difference/mean": 0.30028611421585083, "step": 40 }, { "clip_ratio/high_max": 0.0007337608694797382, "clip_ratio/high_mean": 0.0007337608694797382, "clip_ratio/low_mean": 0.000700917316458395, "clip_ratio/low_min": 0.000700917316458395, "clip_ratio/region_mean": 0.0014346781843212536, "completions/clipped_ratio": 0.00390625, "completions/max_length": 1024.0, "completions/max_terminated_length": 995.0, "completions/mean_length": 73.091796875, "completions/mean_terminated_length": 69.36274719238281, "completions/min_length": 26.0, "completions/min_terminated_length": 26.0, "entropy": 0.7282936986949708, "epoch": 0.01120199146514936, "frac_reward_zero_std": 0.515625, "grad_norm": 1.3456498384475708, "kl": 0.0651899280750917, "learning_rate": 4.952837797873106e-07, "loss": 0.1532, "num_tokens": 4084332.0, "reward": 0.93359375, "reward_std": 0.1842852383852005, "rewards/equation_reward_func/mean": 0.00390625, "rewards/equation_reward_func/std": 0.06243881583213806, "rewards/format_reward_func/mean": 0.9296875, "rewards/format_reward_func/std": 0.25592297315597534, "sampling/importance_sampling_ratio/max": 1.4386967420578003, "sampling/importance_sampling_ratio/mean": 0.9996576309204102, "sampling/importance_sampling_ratio/min": 0.24208980798721313, "sampling/sampling_logp_difference/max": 1.4184465408325195, "sampling/sampling_logp_difference/mean": 0.018967146053910255, "step": 42 }, { "clip_ratio/high_max": 0.0005429589736094284, "clip_ratio/high_mean": 0.0005429589736094284, "clip_ratio/low_mean": 0.0007054127959741487, "clip_ratio/low_min": 0.0007054127959741487, "clip_ratio/region_mean": 0.0012483717776679744, "completions/clipped_ratio": 0.00390625, "completions/max_length": 1024.0, "completions/max_terminated_length": 765.0, "completions/mean_length": 65.826171875, "completions/mean_terminated_length": 62.06863021850586, "completions/min_length": 17.0, "completions/min_terminated_length": 17.0, "entropy": 0.6828638480769263, "epoch": 0.011735419630156473, "frac_reward_zero_std": 0.71875, "grad_norm": 1.0256437063217163, "kl": 0.09110636295129855, "learning_rate": 4.9456184038786e-07, "loss": 0.141, "num_tokens": 4196131.0, "reward": 0.970703125, "reward_std": 0.10905230790376663, "rewards/equation_reward_func/mean": 0.009765625, "rewards/equation_reward_func/std": 0.09843364357948303, "rewards/format_reward_func/mean": 0.9609375, "rewards/format_reward_func/std": 0.1939331740140915, "sampling/importance_sampling_ratio/max": 1.4373838901519775, "sampling/importance_sampling_ratio/mean": 0.9722147583961487, "sampling/importance_sampling_ratio/min": 5.824247334418331e-11, "sampling/sampling_logp_difference/max": 23.56640625, "sampling/sampling_logp_difference/mean": 0.2027144581079483, "step": 44 }, { "clip_ratio/high_max": 0.012247134071205638, "clip_ratio/high_mean": 0.012247134071205638, "clip_ratio/low_mean": 0.000293311591020837, "clip_ratio/low_min": 0.000293311591020837, "clip_ratio/region_mean": 0.012540445503772289, "completions/clipped_ratio": 0.00390625, "completions/max_length": 1024.0, "completions/max_terminated_length": 561.0, "completions/mean_length": 64.818359375, "completions/mean_terminated_length": 61.05686569213867, "completions/min_length": 22.0, "completions/min_terminated_length": 22.0, "entropy": 0.7212139268716177, "epoch": 0.012268847795163585, "frac_reward_zero_std": 0.65625, "grad_norm": 1.3760632276535034, "kl": 1.7077488615694973, "learning_rate": 4.937891121902508e-07, "loss": 0.1133, "num_tokens": 4307502.0, "reward": 0.966796875, "reward_std": 0.13146039843559265, "rewards/equation_reward_func/mean": 0.009765625, "rewards/equation_reward_func/std": 0.09843364357948303, "rewards/format_reward_func/mean": 0.95703125, "rewards/format_reward_func/std": 0.2029850035905838, "sampling/importance_sampling_ratio/max": 1.4471436738967896, "sampling/importance_sampling_ratio/mean": 0.9727644920349121, "sampling/importance_sampling_ratio/min": 6.65283175771414e-11, "sampling/sampling_logp_difference/max": 23.433393478393555, "sampling/sampling_logp_difference/mean": 0.1866401731967926, "step": 46 }, { "clip_ratio/high_max": 0.0002938441725240813, "clip_ratio/high_mean": 0.0002938441725240813, "clip_ratio/low_mean": 0.0002586909201151381, "clip_ratio/low_min": 0.0002586909201151381, "clip_ratio/region_mean": 0.0005525350894054605, "completions/clipped_ratio": 0.0, "completions/max_length": 785.0, "completions/max_terminated_length": 785.0, "completions/mean_length": 55.564453125, "completions/mean_terminated_length": 55.564453125, "completions/min_length": 20.0, "completions/min_terminated_length": 20.0, "entropy": 0.6383216695653068, "epoch": 0.012802275960170697, "frac_reward_zero_std": 0.6875, "grad_norm": 0.8559442162513733, "kl": 0.1444133176571793, "learning_rate": 4.929657556689726e-07, "loss": 0.0403, "num_tokens": 4413927.0, "reward": 0.970703125, "reward_std": 0.11219413578510284, "rewards/equation_reward_func/mean": 0.005859375, "rewards/equation_reward_func/std": 0.07639661431312561, "rewards/format_reward_func/mean": 0.96484375, "rewards/format_reward_func/std": 0.1843547374010086, "sampling/importance_sampling_ratio/max": 1.4029735326766968, "sampling/importance_sampling_ratio/mean": 1.00029456615448, "sampling/importance_sampling_ratio/min": 0.4570175111293793, "sampling/sampling_logp_difference/max": 0.7830336093902588, "sampling/sampling_logp_difference/mean": 0.018120447173714638, "step": 48 }, { "clip_ratio/high_max": 9.298361449812849e-05, "clip_ratio/high_mean": 9.298361449812849e-05, "clip_ratio/low_mean": 0.003963445601079406, "clip_ratio/low_min": 0.003963445601079406, "clip_ratio/region_mean": 0.004056429215577534, "completions/clipped_ratio": 0.001953125, "completions/max_length": 455.5, "completions/max_terminated_length": 455.5, "completions/mean_length": 56.0908203125, "completions/mean_terminated_length": 55.42214393615723, "completions/min_length": 22.0, "completions/min_terminated_length": 22.0, "entropy": 0.703913731707467, "epoch": 0.01333570412517781, "frac_reward_zero_std": 0.8203125, "grad_norm": 1.4378124475479126, "kl": 0.23787370945016542, "learning_rate": 4.920919418126312e-07, "loss": 0.0137, "num_tokens": 4627532.0, "reward": 0.9814453125, "reward_std": 0.06523782201111317, "rewards/equation_reward_func/mean": 0.0029296875, "rewards/equation_reward_func/std": 0.038198307156562805, "rewards/format_reward_func/mean": 0.978515625, "rewards/format_reward_func/std": 0.14252030476927757, "sampling/importance_sampling_ratio/max": 1.7019571661949158, "sampling/importance_sampling_ratio/mean": 0.9878095090389252, "sampling/importance_sampling_ratio/min": 0.2683930397276962, "sampling/sampling_logp_difference/max": 12.1842200756073, "sampling/sampling_logp_difference/mean": 0.1233413452282548, "step": 50 }, { "clip_ratio/high_max": 9.745953138917685e-05, "clip_ratio/high_mean": 9.745953138917685e-05, "clip_ratio/low_mean": 0.00014718516225305697, "clip_ratio/low_min": 0.00014718516225305697, "clip_ratio/region_mean": 0.0002446446936422338, "completions/clipped_ratio": 0.0, "completions/max_length": 441.0, "completions/max_terminated_length": 441.0, "completions/mean_length": 56.39453125, "completions/mean_terminated_length": 56.39453125, "completions/min_length": 25.0, "completions/min_terminated_length": 25.0, "entropy": 0.5998011049297121, "epoch": 0.013869132290184922, "frac_reward_zero_std": 0.859375, "grad_norm": 0.650745153427124, "kl": 0.11571263087292512, "learning_rate": 4.911678520884398e-07, "loss": 0.0339, "num_tokens": 4734494.0, "reward": 0.99609375, "reward_std": 0.051427144557237625, "rewards/equation_reward_func/mean": 0.0078125, "rewards/equation_reward_func/std": 0.08812850713729858, "rewards/format_reward_func/mean": 0.98828125, "rewards/format_reward_func/std": 0.10772226005792618, "sampling/importance_sampling_ratio/max": 1.6142765283584595, "sampling/importance_sampling_ratio/mean": 0.9999446868896484, "sampling/importance_sampling_ratio/min": 0.6179465651512146, "sampling/sampling_logp_difference/max": 0.4813532829284668, "sampling/sampling_logp_difference/mean": 0.01780065894126892, "step": 52 }, { "clip_ratio/high_max": 0.00015658765591474043, "clip_ratio/high_mean": 0.00015658765591474043, "clip_ratio/low_mean": 0.0001687218997782717, "clip_ratio/low_min": 0.0001687218997782717, "clip_ratio/region_mean": 0.00032530955569301214, "completions/clipped_ratio": 0.0, "completions/max_length": 255.0, "completions/max_terminated_length": 255.0, "completions/mean_length": 55.21484375, "completions/mean_terminated_length": 55.21484375, "completions/min_length": 22.0, "completions/min_terminated_length": 22.0, "entropy": 0.5713819041848183, "epoch": 0.014402560455192033, "frac_reward_zero_std": 0.84375, "grad_norm": 0.9380442500114441, "kl": 0.09363365049163501, "learning_rate": 4.901936784045324e-07, "loss": 0.0009, "num_tokens": 4841012.0, "reward": 1.00390625, "reward_std": 0.055242717266082764, "rewards/equation_reward_func/mean": 0.01171875, "rewards/equation_reward_func/std": 0.10772226005792618, "rewards/format_reward_func/mean": 0.9921875, "rewards/format_reward_func/std": 0.08812850713729858, "sampling/importance_sampling_ratio/max": 1.5352394580841064, "sampling/importance_sampling_ratio/mean": 0.9997539520263672, "sampling/importance_sampling_ratio/min": 0.5285897254943848, "sampling/sampling_logp_difference/max": 0.637542724609375, "sampling/sampling_logp_difference/mean": 0.016956791281700134, "step": 54 }, { "clip_ratio/high_max": 8.734321777915789e-05, "clip_ratio/high_mean": 8.734321777915789e-05, "clip_ratio/low_mean": 9.36030507242928e-05, "clip_ratio/low_min": 9.36030507242928e-05, "clip_ratio/region_mean": 0.00018094626850345067, "completions/clipped_ratio": 0.0, "completions/max_length": 116.0, "completions/max_terminated_length": 116.0, "completions/mean_length": 54.68359375, "completions/mean_terminated_length": 54.68359375, "completions/min_length": 26.0, "completions/min_terminated_length": 26.0, "entropy": 0.5553412776854303, "epoch": 0.014935988620199146, "frac_reward_zero_std": 0.90625, "grad_norm": 0.4358917474746704, "kl": 0.10994591563940048, "learning_rate": 4.891696230701103e-07, "loss": 0.0028, "num_tokens": 4947034.0, "reward": 0.998046875, "reward_std": 0.03597327321767807, "rewards/equation_reward_func/mean": 0.005859375, "rewards/equation_reward_func/std": 0.07639661431312561, "rewards/format_reward_func/mean": 0.9921875, "rewards/format_reward_func/std": 0.08812850713729858, "sampling/importance_sampling_ratio/max": 1.7774344682693481, "sampling/importance_sampling_ratio/mean": 0.9994958639144897, "sampling/importance_sampling_ratio/min": 0.5659688115119934, "sampling/sampling_logp_difference/max": 0.5751709938049316, "sampling/sampling_logp_difference/mean": 0.01716645061969757, "step": 56 }, { "clip_ratio/high_max": 3.480924513294465e-05, "clip_ratio/high_mean": 3.480924513294465e-05, "clip_ratio/low_mean": 0.00027445801727784175, "clip_ratio/low_min": 0.00027445801727784175, "clip_ratio/region_mean": 0.0003092672624107864, "completions/clipped_ratio": 0.0, "completions/max_length": 388.0, "completions/max_terminated_length": 388.0, "completions/mean_length": 57.43359375, "completions/mean_terminated_length": 57.43359375, "completions/min_length": 27.0, "completions/min_terminated_length": 27.0, "entropy": 0.5444329058130583, "epoch": 0.015469416785206259, "frac_reward_zero_std": 0.84375, "grad_norm": 0.5821528434753418, "kl": 0.2076412288265096, "learning_rate": 4.880958987534282e-07, "loss": 0.0067, "num_tokens": 5054536.0, "reward": 1.0078125, "reward_std": 0.05973203852772713, "rewards/equation_reward_func/mean": 0.015625, "rewards/equation_reward_func/std": 0.12414088100194931, "rewards/format_reward_func/mean": 0.9921875, "rewards/format_reward_func/std": 0.08812850713729858, "sampling/importance_sampling_ratio/max": 1.5803041458129883, "sampling/importance_sampling_ratio/mean": 0.9993907809257507, "sampling/importance_sampling_ratio/min": 0.5755361318588257, "sampling/sampling_logp_difference/max": 0.5524532794952393, "sampling/sampling_logp_difference/mean": 0.016767196357250214, "step": 58 }, { "clip_ratio/high_max": 0.00018749161649288403, "clip_ratio/high_mean": 0.00018749161649288403, "clip_ratio/low_mean": 0.0004123110233599113, "clip_ratio/low_min": 0.0004123110233599113, "clip_ratio/region_mean": 0.0005998026398527953, "completions/clipped_ratio": 0.0, "completions/max_length": 108.0, "completions/max_terminated_length": 108.0, "completions/mean_length": 57.244140625, "completions/mean_terminated_length": 57.244140625, "completions/min_length": 23.0, "completions/min_terminated_length": 23.0, "entropy": 0.5230812289648585, "epoch": 0.016002844950213372, "frac_reward_zero_std": 0.890625, "grad_norm": 0.5023806691169739, "kl": 0.11861923730207814, "learning_rate": 4.869727284376277e-07, "loss": -0.0022, "num_tokens": 5161789.0, "reward": 1.001953125, "reward_std": 0.038669899106025696, "rewards/equation_reward_func/mean": 0.0078125, "rewards/equation_reward_func/std": 0.08812850713729858, "rewards/format_reward_func/mean": 0.994140625, "rewards/format_reward_func/std": 0.07639661431312561, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.000137209892273, "sampling/importance_sampling_ratio/min": 0.6136685609817505, "sampling/sampling_logp_difference/max": 0.981741189956665, "sampling/sampling_logp_difference/mean": 0.015835393220186234, "step": 60 }, { "clip_ratio/high_max": 0.0001258419652003795, "clip_ratio/high_mean": 0.0001258419652003795, "clip_ratio/low_mean": 0.00020840929937548935, "clip_ratio/low_min": 0.00020840929937548935, "clip_ratio/region_mean": 0.00033425126457586884, "completions/clipped_ratio": 0.0, "completions/max_length": 126.0, "completions/max_terminated_length": 126.0, "completions/mean_length": 59.115234375, "completions/mean_terminated_length": 59.115234375, "completions/min_length": 23.0, "completions/min_terminated_length": 23.0, "entropy": 0.5265028360817168, "epoch": 0.016536273115220483, "frac_reward_zero_std": 0.859375, "grad_norm": 0.5980551242828369, "kl": 0.12793309407101738, "learning_rate": 4.858003453744314e-07, "loss": -0.0025, "num_tokens": 5270296.0, "reward": 1.009765625, "reward_std": 0.05681721866130829, "rewards/equation_reward_func/mean": 0.017578125, "rewards/equation_reward_func/std": 0.13154059648513794, "rewards/format_reward_func/mean": 0.9921875, "rewards/format_reward_func/std": 0.08812850713729858, "sampling/importance_sampling_ratio/max": 1.4510185718536377, "sampling/importance_sampling_ratio/mean": 1.0003494024276733, "sampling/importance_sampling_ratio/min": 0.6126307845115662, "sampling/sampling_logp_difference/max": 0.4899929165840149, "sampling/sampling_logp_difference/mean": 0.016140108928084373, "step": 62 }, { "clip_ratio/high_max": 3.1965222054471574e-05, "clip_ratio/high_mean": 3.1965222054471574e-05, "clip_ratio/low_mean": 9.001686380037831e-05, "clip_ratio/low_min": 9.001686380037831e-05, "clip_ratio/region_mean": 0.00012198208585484988, "completions/clipped_ratio": 0.0, "completions/max_length": 100.0, "completions/max_terminated_length": 100.0, "completions/mean_length": 58.421875, "completions/mean_terminated_length": 58.421875, "completions/min_length": 25.0, "completions/min_terminated_length": 25.0, "entropy": 0.49459916518794167, "epoch": 0.017069701280227598, "frac_reward_zero_std": 0.796875, "grad_norm": 0.653678297996521, "kl": 0.26154189225700164, "learning_rate": 4.845789930357016e-07, "loss": 0.005, "num_tokens": 5378456.0, "reward": 1.01171875, "reward_std": 0.07352422922849655, "rewards/equation_reward_func/mean": 0.01953125, "rewards/equation_reward_func/std": 0.1385180652141571, "rewards/format_reward_func/mean": 0.9921875, "rewards/format_reward_func/std": 0.08812850713729858, "sampling/importance_sampling_ratio/max": 1.4458683729171753, "sampling/importance_sampling_ratio/mean": 1.000059723854065, "sampling/importance_sampling_ratio/min": 0.6116892695426941, "sampling/sampling_logp_difference/max": 0.4915308952331543, "sampling/sampling_logp_difference/mean": 0.015089752152562141, "step": 64 }, { "clip_ratio/high_max": 8.667137848937677e-05, "clip_ratio/high_mean": 8.667137848937677e-05, "clip_ratio/low_mean": 0.00017608674049067, "clip_ratio/low_min": 0.00017608674049067, "clip_ratio/region_mean": 0.00026275812221380573, "completions/clipped_ratio": 0.0009765625, "completions/max_length": 200.5, "completions/max_terminated_length": 200.5, "completions/mean_length": 59.0, "completions/mean_terminated_length": 58.88188171386719, "completions/min_length": 25.0, "completions/min_terminated_length": 25.0, "entropy": 0.5012839436531067, "epoch": 0.01760312944523471, "frac_reward_zero_std": 0.78125, "grad_norm": 0.7769564986228943, "kl": 0.14915075670513842, "learning_rate": 4.833089250628786e-07, "loss": -0.0015, "num_tokens": 5594960.0, "reward": 1.0185546875, "reward_std": 0.0828927680850029, "rewards/equation_reward_func/mean": 0.0263671875, "rewards/equation_reward_func/std": 0.16035238653421402, "rewards/format_reward_func/mean": 0.9921875, "rewards/format_reward_func/std": 0.08508053794503212, "sampling/importance_sampling_ratio/max": 1.5077810287475586, "sampling/importance_sampling_ratio/mean": 0.9981475174427032, "sampling/importance_sampling_ratio/min": 0.2692959010847062, "sampling/sampling_logp_difference/max": 12.176925659179688, "sampling/sampling_logp_difference/mean": 0.0425907289609313, "step": 66 }, { "clip_ratio/high_max": 0.00012458682370682558, "clip_ratio/high_mean": 0.00012458682370682558, "clip_ratio/low_mean": 0.00039236547632349864, "clip_ratio/low_min": 0.00039236547632349864, "clip_ratio/region_mean": 0.0005169522967965653, "completions/clipped_ratio": 0.001953125, "completions/max_length": 1024.0, "completions/max_terminated_length": 173.0, "completions/mean_length": 60.228515625, "completions/mean_terminated_length": 58.342464447021484, "completions/min_length": 30.0, "completions/min_terminated_length": 30.0, "entropy": 0.48701537897189456, "epoch": 0.01813655761024182, "frac_reward_zero_std": 0.796875, "grad_norm": 0.5133281350135803, "kl": 0.14751790546708637, "learning_rate": 4.819904052143058e-07, "loss": 0.0069, "num_tokens": 5703877.0, "reward": 1.015625, "reward_std": 0.08259691298007965, "rewards/equation_reward_func/mean": 0.025390625, "rewards/equation_reward_func/std": 0.15746226906776428, "rewards/format_reward_func/mean": 0.990234375, "rewards/format_reward_func/std": 0.09843364357948303, "sampling/importance_sampling_ratio/max": 1.4501948356628418, "sampling/importance_sampling_ratio/mean": 1.000070571899414, "sampling/importance_sampling_ratio/min": 0.5305090546607971, "sampling/sampling_logp_difference/max": 0.633918285369873, "sampling/sampling_logp_difference/mean": 0.015203745104372501, "step": 68 }, { "clip_ratio/high_max": 0.0001730913564743888, "clip_ratio/high_mean": 0.0001730913564743888, "clip_ratio/low_mean": 0.00018130017108180455, "clip_ratio/low_min": 0.00018130017108180455, "clip_ratio/region_mean": 0.00035439152755619335, "completions/clipped_ratio": 0.0, "completions/max_length": 101.0, "completions/max_terminated_length": 101.0, "completions/mean_length": 57.494140625, "completions/mean_terminated_length": 57.494140625, "completions/min_length": 32.0, "completions/min_terminated_length": 32.0, "entropy": 0.499873752395312, "epoch": 0.018669985775248935, "frac_reward_zero_std": 0.8125, "grad_norm": 0.4106377065181732, "kl": 0.14240571359793344, "learning_rate": 4.806237073104548e-07, "loss": 0.0431, "num_tokens": 5811354.0, "reward": 1.0234375, "reward_std": 0.07082759588956833, "rewards/equation_reward_func/mean": 0.025390625, "rewards/equation_reward_func/std": 0.15746226906776428, "rewards/format_reward_func/mean": 0.998046875, "rewards/format_reward_func/std": 0.04419417306780815, "sampling/importance_sampling_ratio/max": 1.4750500917434692, "sampling/importance_sampling_ratio/mean": 0.9996945261955261, "sampling/importance_sampling_ratio/min": 0.6141188740730286, "sampling/sampling_logp_difference/max": 0.4875667095184326, "sampling/sampling_logp_difference/mean": 0.0145404152572155, "step": 70 }, { "clip_ratio/high_max": 2.7367269568559197e-05, "clip_ratio/high_mean": 2.7367269568559197e-05, "clip_ratio/low_mean": 0.00012311079060762294, "clip_ratio/low_min": 0.00012311079060762294, "clip_ratio/region_mean": 0.00015047806017618213, "completions/clipped_ratio": 0.0, "completions/max_length": 117.0, "completions/max_terminated_length": 117.0, "completions/mean_length": 56.755859375, "completions/mean_terminated_length": 56.755859375, "completions/min_length": 27.0, "completions/min_terminated_length": 27.0, "entropy": 0.48587797333796817, "epoch": 0.019203413940256046, "frac_reward_zero_std": 0.71875, "grad_norm": 0.6676278114318848, "kl": 0.13489198436339697, "learning_rate": 4.792091151770602e-07, "loss": -0.0127, "num_tokens": 5918501.0, "reward": 1.0390625, "reward_std": 0.1152973473072052, "rewards/equation_reward_func/mean": 0.044921875, "rewards/equation_reward_func/std": 0.20733514428138733, "rewards/format_reward_func/mean": 0.994140625, "rewards/format_reward_func/std": 0.07639661431312561, "sampling/importance_sampling_ratio/max": 1.5762015581130981, "sampling/importance_sampling_ratio/mean": 1.0003767013549805, "sampling/importance_sampling_ratio/min": 0.4934070110321045, "sampling/sampling_logp_difference/max": 0.7064208984375, "sampling/sampling_logp_difference/mean": 0.014744394458830357, "step": 72 }, { "clip_ratio/high_max": 9.112760825600062e-05, "clip_ratio/high_mean": 9.112760825600062e-05, "clip_ratio/low_mean": 0.0004944723299963193, "clip_ratio/low_min": 0.0004944723299963193, "clip_ratio/region_mean": 0.0005855999382523199, "completions/clipped_ratio": 0.0, "completions/max_length": 234.0, "completions/max_terminated_length": 234.0, "completions/mean_length": 57.236328125, "completions/mean_terminated_length": 57.236328125, "completions/min_length": 26.0, "completions/min_terminated_length": 26.0, "entropy": 0.4855133162604438, "epoch": 0.019736842105263157, "frac_reward_zero_std": 0.765625, "grad_norm": 0.6810640692710876, "kl": 0.31850574165582657, "learning_rate": 4.777469225861765e-07, "loss": 0.0063, "num_tokens": 6025774.0, "reward": 1.03125, "reward_std": 0.0962078869342804, "rewards/equation_reward_func/mean": 0.0390625, "rewards/equation_reward_func/std": 0.1939331740140915, "rewards/format_reward_func/mean": 0.9921875, "rewards/format_reward_func/std": 0.08812850713729858, "sampling/importance_sampling_ratio/max": 1.6487880945205688, "sampling/importance_sampling_ratio/mean": 0.9998194575309753, "sampling/importance_sampling_ratio/min": 0.5815256834030151, "sampling/sampling_logp_difference/max": 0.5421000719070435, "sampling/sampling_logp_difference/mean": 0.015022373758256435, "step": 74 }, { "clip_ratio/high_max": 9.071986035754283e-05, "clip_ratio/high_mean": 9.071986035754283e-05, "clip_ratio/low_mean": 0.0005455782181040073, "clip_ratio/low_min": 0.0005455782181040073, "clip_ratio/region_mean": 0.0006362980784615502, "completions/clipped_ratio": 0.0, "completions/max_length": 168.0, "completions/max_terminated_length": 168.0, "completions/mean_length": 57.16015625, "completions/mean_terminated_length": 57.16015625, "completions/min_length": 28.0, "completions/min_terminated_length": 28.0, "entropy": 0.4848656083146731, "epoch": 0.02027027027027027, "frac_reward_zero_std": 0.703125, "grad_norm": 0.9179761409759521, "kl": 0.26588521773616475, "learning_rate": 4.762374331951703e-07, "loss": 0.0128, "num_tokens": 6133064.0, "reward": 1.046875, "reward_std": 0.12060342729091644, "rewards/equation_reward_func/mean": 0.05078125, "rewards/equation_reward_func/std": 0.21976542472839355, "rewards/format_reward_func/mean": 0.99609375, "rewards/format_reward_func/std": 0.06243881583213806, "sampling/importance_sampling_ratio/max": 1.4661178588867188, "sampling/importance_sampling_ratio/mean": 0.9996462464332581, "sampling/importance_sampling_ratio/min": 0.5549461841583252, "sampling/sampling_logp_difference/max": 0.5888841152191162, "sampling/sampling_logp_difference/mean": 0.014737057499587536, "step": 76 }, { "clip_ratio/high_max": 0.000124341869700907, "clip_ratio/high_mean": 0.000124341869700907, "clip_ratio/low_mean": 0.00036503980906369787, "clip_ratio/low_min": 0.00036503980906369787, "clip_ratio/region_mean": 0.0004893816755308459, "completions/clipped_ratio": 0.0, "completions/max_length": 91.0, "completions/max_terminated_length": 91.0, "completions/mean_length": 56.6640625, "completions/mean_terminated_length": 56.6640625, "completions/min_length": 25.0, "completions/min_terminated_length": 25.0, "entropy": 0.46769310616784626, "epoch": 0.020803698435277383, "frac_reward_zero_std": 0.8125, "grad_norm": 0.49485430121421814, "kl": 0.17351551829940742, "learning_rate": 4.7468096048365814e-07, "loss": 0.0042, "num_tokens": 6240300.0, "reward": 1.033203125, "reward_std": 0.07595369219779968, "rewards/equation_reward_func/mean": 0.03515625, "rewards/equation_reward_func/std": 0.1843547374010086, "rewards/format_reward_func/mean": 0.998046875, "rewards/format_reward_func/std": 0.04419417306780815, "sampling/importance_sampling_ratio/max": 1.554433822631836, "sampling/importance_sampling_ratio/mean": 0.9997377395629883, "sampling/importance_sampling_ratio/min": 0.6880477070808411, "sampling/sampling_logp_difference/max": 0.44111132621765137, "sampling/sampling_logp_difference/mean": 0.014428908936679363, "step": 78 }, { "clip_ratio/high_max": 9.466569948320587e-05, "clip_ratio/high_mean": 9.466569948320587e-05, "clip_ratio/low_mean": 0.00037429635995067656, "clip_ratio/low_min": 0.00037429635995067656, "clip_ratio/region_mean": 0.0004689620594338824, "completions/clipped_ratio": 0.0, "completions/max_length": 141.0, "completions/max_terminated_length": 141.0, "completions/mean_length": 55.90625, "completions/mean_terminated_length": 55.90625, "completions/min_length": 22.0, "completions/min_terminated_length": 22.0, "entropy": 0.4568544675906499, "epoch": 0.021337126600284494, "frac_reward_zero_std": 0.671875, "grad_norm": 0.8507832288742065, "kl": 0.2776802186336782, "learning_rate": 4.730778276884061e-07, "loss": 0.0046, "num_tokens": 6347020.0, "reward": 1.056640625, "reward_std": 0.13053178787231445, "rewards/equation_reward_func/mean": 0.060546875, "rewards/equation_reward_func/std": 0.2387305200099945, "rewards/format_reward_func/mean": 0.99609375, "rewards/format_reward_func/std": 0.06243881583213806, "sampling/importance_sampling_ratio/max": 1.431517243385315, "sampling/importance_sampling_ratio/mean": 0.9997592568397522, "sampling/importance_sampling_ratio/min": 0.6875678896903992, "sampling/sampling_logp_difference/max": 0.37459468841552734, "sampling/sampling_logp_difference/mean": 0.01453035231679678, "step": 80 }, { "clip_ratio/high_max": 6.149079288459486e-05, "clip_ratio/high_mean": 6.149079288459486e-05, "clip_ratio/low_mean": 0.00043590317040474876, "clip_ratio/low_min": 0.00043590317040474876, "clip_ratio/region_mean": 0.0004973939697568616, "completions/clipped_ratio": 0.0, "completions/max_length": 119.5, "completions/max_terminated_length": 119.5, "completions/mean_length": 54.400390625, "completions/mean_terminated_length": 54.400390625, "completions/min_length": 28.0, "completions/min_terminated_length": 28.0, "entropy": 0.4379849069648319, "epoch": 0.021870554765291608, "frac_reward_zero_std": 0.6875, "grad_norm": 0.6990366578102112, "kl": 0.23277462273836136, "learning_rate": 4.7142836773620227e-07, "loss": 0.0063, "num_tokens": 6559030.0, "reward": 1.06640625, "reward_std": 0.134109266102314, "rewards/equation_reward_func/mean": 0.0693359375, "rewards/equation_reward_func/std": 0.2542656809091568, "rewards/format_reward_func/mean": 0.9970703125, "rewards/format_reward_func/std": 0.053316494449973106, "sampling/importance_sampling_ratio/max": 1.5191975831985474, "sampling/importance_sampling_ratio/mean": 1.000219702720642, "sampling/importance_sampling_ratio/min": 0.5883514285087585, "sampling/sampling_logp_difference/max": 0.531247615814209, "sampling/sampling_logp_difference/mean": 0.013361877296119928, "step": 82 }, { "clip_ratio/high_max": 6.224729845093357e-05, "clip_ratio/high_mean": 6.224729845093357e-05, "clip_ratio/low_mean": 0.0009572154344318227, "clip_ratio/low_min": 0.0009572154344318227, "clip_ratio/region_mean": 0.0010194627199477206, "completions/clipped_ratio": 0.0, "completions/max_length": 86.0, "completions/max_terminated_length": 86.0, "completions/mean_length": 53.50390625, "completions/mean_terminated_length": 53.50390625, "completions/min_length": 27.0, "completions/min_terminated_length": 27.0, "entropy": 0.4128892694910367, "epoch": 0.02240398293029872, "frac_reward_zero_std": 0.71875, "grad_norm": 0.6463724374771118, "kl": 0.19013702952199513, "learning_rate": 4.6973292317471635e-07, "loss": 0.0052, "num_tokens": 6664664.0, "reward": 1.05078125, "reward_std": 0.11812376976013184, "rewards/equation_reward_func/mean": 0.056640625, "rewards/equation_reward_func/std": 0.23138070106506348, "rewards/format_reward_func/mean": 0.994140625, "rewards/format_reward_func/std": 0.07639661431312561, "sampling/importance_sampling_ratio/max": 1.5252774953842163, "sampling/importance_sampling_ratio/mean": 0.9998437762260437, "sampling/importance_sampling_ratio/min": 0.6066035628318787, "sampling/sampling_logp_difference/max": 0.4998798370361328, "sampling/sampling_logp_difference/mean": 0.012673393823206425, "step": 84 }, { "clip_ratio/high_max": 0.0001963357240634246, "clip_ratio/high_mean": 0.0001963357240634246, "clip_ratio/low_mean": 0.0004532367456704378, "clip_ratio/low_min": 0.0004532367456704378, "clip_ratio/region_mean": 0.0006495724697338624, "completions/clipped_ratio": 0.0, "completions/max_length": 89.0, "completions/max_terminated_length": 89.0, "completions/mean_length": 53.46484375, "completions/mean_terminated_length": 53.46484375, "completions/min_length": 26.0, "completions/min_terminated_length": 26.0, "entropy": 0.39042702731158996, "epoch": 0.02293741109530583, "frac_reward_zero_std": 0.734375, "grad_norm": 0.731373131275177, "kl": 0.3146343932797511, "learning_rate": 4.679918461013627e-07, "loss": -0.0157, "num_tokens": 6770158.0, "reward": 1.064453125, "reward_std": 0.11969826370477676, "rewards/equation_reward_func/mean": 0.068359375, "rewards/equation_reward_func/std": 0.25260838866233826, "rewards/format_reward_func/mean": 0.99609375, "rewards/format_reward_func/std": 0.06243881583213806, "sampling/importance_sampling_ratio/max": 1.8587133884429932, "sampling/importance_sampling_ratio/mean": 0.9997268319129944, "sampling/importance_sampling_ratio/min": 0.6095913052558899, "sampling/sampling_logp_difference/max": 0.6198844909667969, "sampling/sampling_logp_difference/mean": 0.012392060831189156, "step": 86 }, { "clip_ratio/high_max": 6.641253841937416e-05, "clip_ratio/high_mean": 6.641253841937416e-05, "clip_ratio/low_mean": 0.0001936533662956208, "clip_ratio/low_min": 0.0001936533662956208, "clip_ratio/region_mean": 0.00026006590471499495, "completions/clipped_ratio": 0.0, "completions/max_length": 91.0, "completions/max_terminated_length": 91.0, "completions/mean_length": 53.654296875, "completions/mean_terminated_length": 53.654296875, "completions/min_length": 26.0, "completions/min_terminated_length": 26.0, "entropy": 0.3859163332316611, "epoch": 0.023470839260312945, "frac_reward_zero_std": 0.609375, "grad_norm": 0.872736394405365, "kl": 0.19867314977778328, "learning_rate": 4.6620549809017885e-07, "loss": 0.0085, "num_tokens": 6875797.0, "reward": 1.0859375, "reward_std": 0.16314628720283508, "rewards/equation_reward_func/mean": 0.0859375, "rewards/equation_reward_func/std": 0.28054583072662354, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.586866021156311, "sampling/importance_sampling_ratio/mean": 1.0001603364944458, "sampling/importance_sampling_ratio/min": 0.6841540336608887, "sampling/sampling_logp_difference/max": 0.4617609977722168, "sampling/sampling_logp_difference/mean": 0.01188343670219183, "step": 88 }, { "clip_ratio/high_max": 0.0002847820578608662, "clip_ratio/high_mean": 0.0002847820578608662, "clip_ratio/low_mean": 0.00031860194091374677, "clip_ratio/low_min": 0.00031860194091374677, "clip_ratio/region_mean": 0.0006033839923070951, "completions/clipped_ratio": 0.0, "completions/max_length": 91.0, "completions/max_terminated_length": 91.0, "completions/mean_length": 54.541015625, "completions/mean_terminated_length": 54.541015625, "completions/min_length": 30.0, "completions/min_terminated_length": 30.0, "entropy": 0.38282785564661026, "epoch": 0.024004267425320056, "frac_reward_zero_std": 0.75, "grad_norm": 0.8757621049880981, "kl": 0.19160300720896986, "learning_rate": 4.643742501167366e-07, "loss": -0.0029, "num_tokens": 6981970.0, "reward": 1.064453125, "reward_std": 0.10797597467899323, "rewards/equation_reward_func/mean": 0.06640625, "rewards/equation_reward_func/std": 0.2492343932390213, "rewards/format_reward_func/mean": 0.998046875, "rewards/format_reward_func/std": 0.04419417306780815, "sampling/importance_sampling_ratio/max": 1.4009743928909302, "sampling/importance_sampling_ratio/mean": 1.0000730752944946, "sampling/importance_sampling_ratio/min": 0.6123918890953064, "sampling/sampling_logp_difference/max": 0.4903829097747803, "sampling/sampling_logp_difference/mean": 0.01211240142583847, "step": 90 }, { "clip_ratio/high_max": 6.413285478225185e-05, "clip_ratio/high_mean": 6.413285478225185e-05, "clip_ratio/low_mean": 0.00023037084287756847, "clip_ratio/low_min": 0.00023037084287756847, "clip_ratio/region_mean": 0.0002945036976598203, "completions/clipped_ratio": 0.0, "completions/max_length": 81.0, "completions/max_terminated_length": 81.0, "completions/mean_length": 53.505859375, "completions/mean_terminated_length": 53.505859375, "completions/min_length": 29.0, "completions/min_terminated_length": 29.0, "entropy": 0.3810688671138551, "epoch": 0.02453769559032717, "frac_reward_zero_std": 0.671875, "grad_norm": 0.6786807775497437, "kl": 0.23459011399083668, "learning_rate": 4.624984824811006e-07, "loss": 0.011, "num_tokens": 7087429.0, "reward": 1.06640625, "reward_std": 0.13730725646018982, "rewards/equation_reward_func/mean": 0.068359375, "rewards/equation_reward_func/std": 0.25260838866233826, "rewards/format_reward_func/mean": 0.998046875, "rewards/format_reward_func/std": 0.04419417306780815, "sampling/importance_sampling_ratio/max": 1.601763367652893, "sampling/importance_sampling_ratio/mean": 0.9997592568397522, "sampling/importance_sampling_ratio/min": 0.4768765866756439, "sampling/sampling_logp_difference/max": 0.7404975891113281, "sampling/sampling_logp_difference/mean": 0.011574014090001583, "step": 92 }, { "clip_ratio/high_max": 0.0001584748258917696, "clip_ratio/high_mean": 0.0001584748258917696, "clip_ratio/low_mean": 0.00031917635674795345, "clip_ratio/low_min": 0.00031917635674795345, "clip_ratio/region_mean": 0.000477651182639723, "completions/clipped_ratio": 0.0, "completions/max_length": 127.0, "completions/max_terminated_length": 127.0, "completions/mean_length": 55.142578125, "completions/mean_terminated_length": 55.142578125, "completions/min_length": 26.0, "completions/min_terminated_length": 26.0, "entropy": 0.373777824971411, "epoch": 0.025071123755334282, "frac_reward_zero_std": 0.5625, "grad_norm": 0.6518265008926392, "kl": 0.177696091019445, "learning_rate": 4.605785847288502e-07, "loss": -0.0148, "num_tokens": 7193918.0, "reward": 1.0859375, "reward_std": 0.17715543508529663, "rewards/equation_reward_func/mean": 0.0859375, "rewards/equation_reward_func/std": 0.28054583072662354, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.503134846687317, "sampling/importance_sampling_ratio/mean": 0.9995354413986206, "sampling/importance_sampling_ratio/min": 0.5218640565872192, "sampling/sampling_logp_difference/max": 0.6503481864929199, "sampling/sampling_logp_difference/mean": 0.011570579372346401, "step": 94 }, { "clip_ratio/high_max": 9.217362902644608e-05, "clip_ratio/high_mean": 9.217362902644608e-05, "clip_ratio/low_mean": 0.0005733486096788612, "clip_ratio/low_min": 0.0005733486096788612, "clip_ratio/region_mean": 0.0006655222419390662, "completions/clipped_ratio": 0.0, "completions/max_length": 97.0, "completions/max_terminated_length": 97.0, "completions/mean_length": 54.259765625, "completions/mean_terminated_length": 54.259765625, "completions/min_length": 28.0, "completions/min_terminated_length": 28.0, "entropy": 0.3661065283748839, "epoch": 0.025604551920341393, "frac_reward_zero_std": 0.65625, "grad_norm": 0.7916492819786072, "kl": 0.3318152477343877, "learning_rate": 4.5861495557018206e-07, "loss": 0.0019, "num_tokens": 7299955.0, "reward": 1.072265625, "reward_std": 0.14144614338874817, "rewards/equation_reward_func/mean": 0.07421875, "rewards/equation_reward_func/std": 0.2623828947544098, "rewards/format_reward_func/mean": 0.998046875, "rewards/format_reward_func/std": 0.04419417306780815, "sampling/importance_sampling_ratio/max": 1.4506229162216187, "sampling/importance_sampling_ratio/mean": 0.9998356103897095, "sampling/importance_sampling_ratio/min": 0.6574238538742065, "sampling/sampling_logp_difference/max": 0.4194263219833374, "sampling/sampling_logp_difference/mean": 0.011709309183061123, "step": 96 }, { "clip_ratio/high_max": 6.440817055085467e-05, "clip_ratio/high_mean": 6.440817055085467e-05, "clip_ratio/low_mean": 0.0001222810533363372, "clip_ratio/low_min": 0.0001222810533363372, "clip_ratio/region_mean": 0.00018668922388719188, "completions/clipped_ratio": 0.0, "completions/max_length": 125.0, "completions/max_terminated_length": 125.0, "completions/mean_length": 54.677734375, "completions/mean_terminated_length": 54.677734375, "completions/min_length": 29.5, "completions/min_terminated_length": 29.5, "entropy": 0.342889907459418, "epoch": 0.026137980085348508, "frac_reward_zero_std": 0.6171875, "grad_norm": 0.7152344584465027, "kl": 0.24752955407732063, "learning_rate": 4.566080027971082e-07, "loss": 0.0083, "num_tokens": 7512065.0, "reward": 1.1142578125, "reward_std": 0.17356323450803757, "rewards/equation_reward_func/mean": 0.1171875, "rewards/equation_reward_func/std": 0.3217283636331558, "rewards/format_reward_func/mean": 0.9970703125, "rewards/format_reward_func/std": 0.053316494449973106, "sampling/importance_sampling_ratio/max": 1.4866527318954468, "sampling/importance_sampling_ratio/mean": 1.0000663995742798, "sampling/importance_sampling_ratio/min": 0.6695625483989716, "sampling/sampling_logp_difference/max": 0.42328375577926636, "sampling/sampling_logp_difference/mean": 0.010475398041307926, "step": 98 }, { "clip_ratio/high_max": 0.00018635201720624335, "clip_ratio/high_mean": 0.00018635201720624335, "clip_ratio/low_mean": 0.00033971545700397756, "clip_ratio/low_min": 0.00033971545700397756, "clip_ratio/region_mean": 0.0005260674742102209, "completions/clipped_ratio": 0.0, "completions/max_length": 80.0, "completions/max_terminated_length": 80.0, "completions/mean_length": 54.22265625, "completions/mean_terminated_length": 54.22265625, "completions/min_length": 30.0, "completions/min_terminated_length": 30.0, "entropy": 0.3336399926079644, "epoch": 0.02667140825035562, "frac_reward_zero_std": 0.578125, "grad_norm": 0.752447247505188, "kl": 0.22766229386130968, "learning_rate": 4.545581431987694e-07, "loss": -0.0057, "num_tokens": 7617915.0, "reward": 1.123046875, "reward_std": 0.182941734790802, "rewards/equation_reward_func/mean": 0.123046875, "rewards/equation_reward_func/std": 0.32881227135658264, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4538755416870117, "sampling/importance_sampling_ratio/mean": 1.000040888786316, "sampling/importance_sampling_ratio/min": 0.6096076369285583, "sampling/sampling_logp_difference/max": 0.49493980407714844, "sampling/sampling_logp_difference/mean": 0.010414124466478825, "step": 100 }, { "clip_ratio/high_max": 9.638364685492383e-05, "clip_ratio/high_mean": 9.638364685492383e-05, "clip_ratio/low_mean": 0.000633861927781254, "clip_ratio/low_min": 0.000633861927781254, "clip_ratio/region_mean": 0.0007302455778699368, "completions/clipped_ratio": 0.0, "completions/max_length": 84.0, "completions/max_terminated_length": 84.0, "completions/mean_length": 55.734375, "completions/mean_terminated_length": 55.734375, "completions/min_length": 30.0, "completions/min_terminated_length": 30.0, "entropy": 0.31998403122027713, "epoch": 0.02720483641536273, "frac_reward_zero_std": 0.609375, "grad_norm": 0.8397956490516663, "kl": 0.30809418112039566, "learning_rate": 4.5246580247487933e-07, "loss": -0.0068, "num_tokens": 7724659.0, "reward": 1.134765625, "reward_std": 0.1721571683883667, "rewards/equation_reward_func/mean": 0.134765625, "rewards/equation_reward_func/std": 0.3418070077896118, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.3653279542922974, "sampling/importance_sampling_ratio/mean": 1.0000591278076172, "sampling/importance_sampling_ratio/min": 0.6534257531166077, "sampling/sampling_logp_difference/max": 0.42552638053894043, "sampling/sampling_logp_difference/mean": 0.010140168480575085, "step": 102 }, { "clip_ratio/high_max": 3.0491523729223343e-05, "clip_ratio/high_mean": 3.0491523729223343e-05, "clip_ratio/low_mean": 0.00024429223978788487, "clip_ratio/low_min": 0.00024429223978788487, "clip_ratio/region_mean": 0.0002747837635171082, "completions/clipped_ratio": 0.0, "completions/max_length": 82.0, "completions/max_terminated_length": 82.0, "completions/mean_length": 53.490234375, "completions/mean_terminated_length": 53.490234375, "completions/min_length": 27.0, "completions/min_terminated_length": 27.0, "entropy": 0.3079003418485324, "epoch": 0.027738264580369845, "frac_reward_zero_std": 0.65625, "grad_norm": 0.663601279258728, "kl": 0.22134189742306867, "learning_rate": 4.5033141514731786e-07, "loss": 0.0185, "num_tokens": 7830126.0, "reward": 1.103515625, "reward_std": 0.14598248898983002, "rewards/equation_reward_func/mean": 0.103515625, "rewards/equation_reward_func/std": 0.30492907762527466, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.8041462898254395, "sampling/importance_sampling_ratio/mean": 1.0000555515289307, "sampling/importance_sampling_ratio/min": 0.47461166977882385, "sampling/sampling_logp_difference/max": 0.7452583312988281, "sampling/sampling_logp_difference/mean": 0.010407421737909317, "step": 104 }, { "clip_ratio/high_max": 0.00012673301454116072, "clip_ratio/high_mean": 0.00012673301454116072, "clip_ratio/low_mean": 0.0003165318339597434, "clip_ratio/low_min": 0.0003165318339597434, "clip_ratio/region_mean": 0.0004432648485009041, "completions/clipped_ratio": 0.0, "completions/max_length": 96.0, "completions/max_terminated_length": 96.0, "completions/mean_length": 54.42578125, "completions/mean_terminated_length": 54.42578125, "completions/min_length": 29.0, "completions/min_terminated_length": 29.0, "entropy": 0.30098287926779854, "epoch": 0.028271692745376956, "frac_reward_zero_std": 0.671875, "grad_norm": 0.621425211429596, "kl": 0.2817885786708858, "learning_rate": 4.4815542446989373e-07, "loss": -0.0071, "num_tokens": 7936128.0, "reward": 1.109375, "reward_std": 0.14643804728984833, "rewards/equation_reward_func/mean": 0.111328125, "rewards/equation_reward_func/std": 0.31484565138816833, "rewards/format_reward_func/mean": 0.998046875, "rewards/format_reward_func/std": 0.04419417306780815, "sampling/importance_sampling_ratio/max": 1.9009522199630737, "sampling/importance_sampling_ratio/mean": 1.0000678300857544, "sampling/importance_sampling_ratio/min": 0.606683075428009, "sampling/sampling_logp_difference/max": 0.6423549652099609, "sampling/sampling_logp_difference/mean": 0.009736377745866776, "step": 106 }, { "clip_ratio/high_max": 9.985202794066734e-05, "clip_ratio/high_mean": 9.985202794066734e-05, "clip_ratio/low_mean": 0.0005190472616555376, "clip_ratio/low_min": 0.0005190472616555376, "clip_ratio/region_mean": 0.0006188992928299638, "completions/clipped_ratio": 0.0, "completions/max_length": 77.0, "completions/max_terminated_length": 77.0, "completions/mean_length": 53.552734375, "completions/mean_terminated_length": 53.552734375, "completions/min_length": 31.0, "completions/min_terminated_length": 31.0, "entropy": 0.2843737068275611, "epoch": 0.028805120910384067, "frac_reward_zero_std": 0.546875, "grad_norm": 0.629451334476471, "kl": 0.21802533811165226, "learning_rate": 4.4593828233629214e-07, "loss": 0.0029, "num_tokens": 8041515.0, "reward": 1.150390625, "reward_std": 0.18977847695350647, "rewards/equation_reward_func/mean": 0.150390625, "rewards/equation_reward_func/std": 0.35780346393585205, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.5413179397583008, "sampling/importance_sampling_ratio/mean": 0.9999287724494934, "sampling/importance_sampling_ratio/min": 0.6856960654258728, "sampling/sampling_logp_difference/max": 0.43263787031173706, "sampling/sampling_logp_difference/mean": 0.009411610662937164, "step": 108 }, { "clip_ratio/high_max": 6.729989804120528e-05, "clip_ratio/high_mean": 6.729989804120528e-05, "clip_ratio/low_mean": 0.0003927987385799901, "clip_ratio/low_min": 0.0003927987385799901, "clip_ratio/region_mean": 0.0004600986398549543, "completions/clipped_ratio": 0.0, "completions/max_length": 83.0, "completions/max_terminated_length": 83.0, "completions/mean_length": 52.8359375, "completions/mean_terminated_length": 52.8359375, "completions/min_length": 24.0, "completions/min_terminated_length": 24.0, "entropy": 0.2845468541814221, "epoch": 0.02933854907539118, "frac_reward_zero_std": 0.5625, "grad_norm": 0.697964608669281, "kl": 0.24136662897136477, "learning_rate": 4.4368044918622893e-07, "loss": 0.0022, "num_tokens": 8146455.0, "reward": 1.130859375, "reward_std": 0.18825891613960266, "rewards/equation_reward_func/mean": 0.130859375, "rewards/equation_reward_func/std": 0.33757632970809937, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4358296394348145, "sampling/importance_sampling_ratio/mean": 0.9999491572380066, "sampling/importance_sampling_ratio/min": 0.60647052526474, "sampling/sampling_logp_difference/max": 0.5000991821289062, "sampling/sampling_logp_difference/mean": 0.009553857147693634, "step": 110 }, { "clip_ratio/high_max": 0.00022822847434629998, "clip_ratio/high_mean": 0.00022822847434629998, "clip_ratio/low_mean": 0.0002550547134079453, "clip_ratio/low_min": 0.0002550547134079453, "clip_ratio/region_mean": 0.00048328318775424524, "completions/clipped_ratio": 0.0, "completions/max_length": 91.0, "completions/max_terminated_length": 91.0, "completions/mean_length": 54.2265625, "completions/mean_terminated_length": 54.2265625, "completions/min_length": 27.0, "completions/min_terminated_length": 27.0, "entropy": 0.27530702037943733, "epoch": 0.029871977240398292, "frac_reward_zero_std": 0.578125, "grad_norm": 0.7769265174865723, "kl": 0.22397846666475138, "learning_rate": 4.4138239390983e-07, "loss": -0.0023, "num_tokens": 8252491.0, "reward": 1.099609375, "reward_std": 0.17675849795341492, "rewards/equation_reward_func/mean": 0.10546875, "rewards/equation_reward_func/std": 0.3074568510055542, "rewards/format_reward_func/mean": 0.994140625, "rewards/format_reward_func/std": 0.07639661431312561, "sampling/importance_sampling_ratio/max": 1.4896862506866455, "sampling/importance_sampling_ratio/mean": 0.9999532103538513, "sampling/importance_sampling_ratio/min": 0.6068167686462402, "sampling/sampling_logp_difference/max": 0.4995284080505371, "sampling/sampling_logp_difference/mean": 0.008937017060816288, "step": 112 }, { "clip_ratio/high_max": 3.2603026031413014e-05, "clip_ratio/high_mean": 3.2603026031413014e-05, "clip_ratio/low_mean": 6.739244822205769e-05, "clip_ratio/low_min": 6.739244822205769e-05, "clip_ratio/region_mean": 9.99954742534707e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 86.0, "completions/max_terminated_length": 86.0, "completions/mean_length": 52.9208984375, "completions/mean_terminated_length": 52.9208984375, "completions/min_length": 24.5, "completions/min_terminated_length": 24.5, "entropy": 0.2652134576605426, "epoch": 0.030405405405405407, "frac_reward_zero_std": 0.6875, "grad_norm": 0.6283072233200073, "kl": 0.3024534620344639, "learning_rate": 4.390445937502557e-07, "loss": -0.0049, "num_tokens": 8463026.0, "reward": 1.119140625, "reward_std": 0.14075149968266487, "rewards/equation_reward_func/mean": 0.12109375, "rewards/equation_reward_func/std": 0.3256715089082718, "rewards/format_reward_func/mean": 0.998046875, "rewards/format_reward_func/std": 0.03121940791606903, "sampling/importance_sampling_ratio/max": 1.754223883152008, "sampling/importance_sampling_ratio/mean": 1.0002116560935974, "sampling/importance_sampling_ratio/min": 0.5659388899803162, "sampling/sampling_logp_difference/max": 0.6350482702255249, "sampling/sampling_logp_difference/mean": 0.008850776590406895, "step": 114 }, { "clip_ratio/high_max": 0.0001623934529359556, "clip_ratio/high_mean": 0.0001623934529359556, "clip_ratio/low_mean": 0.00040348145153580443, "clip_ratio/low_min": 0.00040348145153580443, "clip_ratio/region_mean": 0.00056587490447176, "completions/clipped_ratio": 0.0, "completions/max_length": 85.0, "completions/max_terminated_length": 85.0, "completions/mean_length": 52.896484375, "completions/mean_terminated_length": 52.896484375, "completions/min_length": 27.0, "completions/min_terminated_length": 27.0, "entropy": 0.25305818766355515, "epoch": 0.030938833570412518, "frac_reward_zero_std": 0.71875, "grad_norm": 0.6711949110031128, "kl": 0.22686809280680287, "learning_rate": 4.3666753420459023e-07, "loss": 0.0053, "num_tokens": 8568341.0, "reward": 1.103515625, "reward_std": 0.12191149592399597, "rewards/equation_reward_func/mean": 0.10546875, "rewards/equation_reward_func/std": 0.3074568510055542, "rewards/format_reward_func/mean": 0.998046875, "rewards/format_reward_func/std": 0.04419417306780815, "sampling/importance_sampling_ratio/max": 1.4110792875289917, "sampling/importance_sampling_ratio/mean": 1.0000543594360352, "sampling/importance_sampling_ratio/min": 0.5361621379852295, "sampling/sampling_logp_difference/max": 0.6233186721801758, "sampling/sampling_logp_difference/mean": 0.008643017150461674, "step": 116 }, { "clip_ratio/high_max": 9.980153926234279e-05, "clip_ratio/high_mean": 9.980153926234279e-05, "clip_ratio/low_mean": 6.652296805340383e-05, "clip_ratio/low_min": 6.652296805340383e-05, "clip_ratio/region_mean": 0.00016632450731574662, "completions/clipped_ratio": 0.0, "completions/max_length": 82.0, "completions/max_terminated_length": 82.0, "completions/mean_length": 52.845703125, "completions/mean_terminated_length": 52.845703125, "completions/min_length": 28.0, "completions/min_terminated_length": 28.0, "entropy": 0.2411729163593716, "epoch": 0.03147226173541963, "frac_reward_zero_std": 0.703125, "grad_norm": 0.7250849008560181, "kl": 0.22534720102945963, "learning_rate": 4.3425170892301764e-07, "loss": 0.0033, "num_tokens": 8673494.0, "reward": 1.09765625, "reward_std": 0.12994009256362915, "rewards/equation_reward_func/mean": 0.1015625, "rewards/equation_reward_func/std": 0.30236753821372986, "rewards/format_reward_func/mean": 0.99609375, "rewards/format_reward_func/std": 0.06243881583213806, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0001612901687622, "sampling/importance_sampling_ratio/min": 0.670235276222229, "sampling/sampling_logp_difference/max": 0.8719024062156677, "sampling/sampling_logp_difference/mean": 0.008650041185319424, "step": 118 }, { "clip_ratio/high_max": 3.200204810127616e-05, "clip_ratio/high_mean": 3.200204810127616e-05, "clip_ratio/low_mean": 0.00013157916772696708, "clip_ratio/low_min": 0.00013157916772696708, "clip_ratio/region_mean": 0.00016358121582824324, "completions/clipped_ratio": 0.0, "completions/max_length": 85.0, "completions/max_terminated_length": 85.0, "completions/mean_length": 52.23046875, "completions/mean_terminated_length": 52.23046875, "completions/min_length": 30.0, "completions/min_terminated_length": 30.0, "entropy": 0.23818034513129127, "epoch": 0.032005689900426744, "frac_reward_zero_std": 0.609375, "grad_norm": 0.626802384853363, "kl": 0.20635693561699656, "learning_rate": 4.3179761960630357e-07, "loss": -0.0024, "num_tokens": 8778276.0, "reward": 1.142578125, "reward_std": 0.17195375263690948, "rewards/equation_reward_func/mean": 0.142578125, "rewards/equation_reward_func/std": 0.3499840497970581, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.3508808612823486, "sampling/importance_sampling_ratio/mean": 1.0002026557922363, "sampling/importance_sampling_ratio/min": 0.5984481573104858, "sampling/sampling_logp_difference/max": 0.5134154558181763, "sampling/sampling_logp_difference/mean": 0.008179975673556328, "step": 120 }, { "clip_ratio/high_max": 0.0001628922182135284, "clip_ratio/high_mean": 0.0001628922182135284, "clip_ratio/low_mean": 0.00015749750632999672, "clip_ratio/low_min": 0.00015749750632999672, "clip_ratio/region_mean": 0.00032038972454352514, "completions/clipped_ratio": 0.0, "completions/max_length": 98.0, "completions/max_terminated_length": 98.0, "completions/mean_length": 53.482421875, "completions/mean_terminated_length": 53.482421875, "completions/min_length": 32.0, "completions/min_terminated_length": 32.0, "entropy": 0.2308905753824446, "epoch": 0.03253911806543385, "frac_reward_zero_std": 0.765625, "grad_norm": 0.540390133857727, "kl": 0.2013553927342097, "learning_rate": 4.293057759016063e-07, "loss": -0.0017, "num_tokens": 8883955.0, "reward": 1.095703125, "reward_std": 0.10389640927314758, "rewards/equation_reward_func/mean": 0.095703125, "rewards/equation_reward_func/std": 0.2944713830947876, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.3597885370254517, "sampling/importance_sampling_ratio/mean": 0.9998118877410889, "sampling/importance_sampling_ratio/min": 0.5017064213752747, "sampling/sampling_logp_difference/max": 0.6897401809692383, "sampling/sampling_logp_difference/mean": 0.008392934687435627, "step": 122 }, { "clip_ratio/high_max": 0.00029407673799950216, "clip_ratio/high_mean": 0.00029407673799950216, "clip_ratio/low_mean": 0.00013097736842205955, "clip_ratio/low_min": 0.00013097736842205955, "clip_ratio/region_mean": 0.0004250541064215617, "completions/clipped_ratio": 0.0, "completions/max_length": 76.0, "completions/max_terminated_length": 76.0, "completions/mean_length": 53.802734375, "completions/mean_terminated_length": 53.802734375, "completions/min_length": 32.0, "completions/min_terminated_length": 32.0, "entropy": 0.2228163286215729, "epoch": 0.033072546230440966, "frac_reward_zero_std": 0.640625, "grad_norm": 0.571448802947998, "kl": 0.2501193769276142, "learning_rate": 4.2677669529663686e-07, "loss": -0.0009, "num_tokens": 8989758.0, "reward": 1.134765625, "reward_std": 0.1630825400352478, "rewards/equation_reward_func/mean": 0.134765625, "rewards/equation_reward_func/std": 0.3418070077896118, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4524681568145752, "sampling/importance_sampling_ratio/mean": 1.000288963317871, "sampling/importance_sampling_ratio/min": 0.6752034425735474, "sampling/sampling_logp_difference/max": 0.39274129271507263, "sampling/sampling_logp_difference/mean": 0.00822390615940094, "step": 124 }, { "clip_ratio/high_max": 9.728386066854e-05, "clip_ratio/high_mean": 9.728386066854e-05, "clip_ratio/low_mean": 0.00012625338988275162, "clip_ratio/low_min": 0.00012625338988275162, "clip_ratio/region_mean": 0.00022353725055129163, "completions/clipped_ratio": 0.0, "completions/max_length": 80.0, "completions/max_terminated_length": 80.0, "completions/mean_length": 53.658203125, "completions/mean_terminated_length": 53.658203125, "completions/min_length": 31.0, "completions/min_terminated_length": 31.0, "entropy": 0.21811328331629434, "epoch": 0.03360597439544808, "frac_reward_zero_std": 0.65625, "grad_norm": 0.6349706649780273, "kl": 0.2652018351687325, "learning_rate": 4.2421090301219077e-07, "loss": -0.0099, "num_tokens": 9095479.0, "reward": 1.103515625, "reward_std": 0.15132677555084229, "rewards/equation_reward_func/mean": 0.10546875, "rewards/equation_reward_func/std": 0.3074568510055542, "rewards/format_reward_func/mean": 0.998046875, "rewards/format_reward_func/std": 0.04419417306780815, "sampling/importance_sampling_ratio/max": 1.3080748319625854, "sampling/importance_sampling_ratio/mean": 0.9998948574066162, "sampling/importance_sampling_ratio/min": 0.5621025562286377, "sampling/sampling_logp_difference/max": 0.57607102394104, "sampling/sampling_logp_difference/mean": 0.007956698536872864, "step": 126 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 9.971219373659955e-05, "clip_ratio/low_min": 9.971219373659955e-05, "clip_ratio/region_mean": 9.971219373659955e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 249.0, "completions/max_terminated_length": 249.0, "completions/mean_length": 53.609375, "completions/mean_terminated_length": 53.609375, "completions/min_length": 32.0, "completions/min_terminated_length": 32.0, "entropy": 0.20628702764709791, "epoch": 0.034139402560455195, "frac_reward_zero_std": 0.6875, "grad_norm": 0.5552213191986084, "kl": 0.19269147598081166, "learning_rate": 4.216089318930741e-07, "loss": 0.0147, "num_tokens": 9201079.0, "reward": 1.09765625, "reward_std": 0.14117899537086487, "rewards/equation_reward_func/mean": 0.099609375, "rewards/equation_reward_func/std": 0.29977133870124817, "rewards/format_reward_func/mean": 0.998046875, "rewards/format_reward_func/std": 0.04419417306780815, "sampling/importance_sampling_ratio/max": 1.4223767518997192, "sampling/importance_sampling_ratio/mean": 0.9998998641967773, "sampling/importance_sampling_ratio/min": 0.4869205355644226, "sampling/sampling_logp_difference/max": 0.7196543216705322, "sampling/sampling_logp_difference/mean": 0.007629196625202894, "step": 128 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00010101531127778192, "clip_ratio/low_min": 0.00010101531127778192, "clip_ratio/region_mean": 0.00010101531127778192, "completions/clipped_ratio": 0.0, "completions/max_length": 85.5, "completions/max_terminated_length": 85.5, "completions/mean_length": 53.05859375, "completions/mean_terminated_length": 53.05859375, "completions/min_length": 31.0, "completions/min_terminated_length": 31.0, "entropy": 0.19540323979324764, "epoch": 0.0346728307254623, "frac_reward_zero_std": 0.640625, "grad_norm": 0.5377609133720398, "kl": 0.3826492495006985, "learning_rate": 4.189713222974466e-07, "loss": 0.0021, "num_tokens": 9411475.0, "reward": 1.1396484375, "reward_std": 0.15932666510343552, "rewards/equation_reward_func/mean": 0.1396484375, "rewards/equation_reward_func/std": 0.34543485939502716, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.6392115950584412, "sampling/importance_sampling_ratio/mean": 1.0000347197055817, "sampling/importance_sampling_ratio/min": 0.5813578218221664, "sampling/sampling_logp_difference/max": 0.7083454132080078, "sampling/sampling_logp_difference/mean": 0.007557510165497661, "step": 130 }, { "clip_ratio/high_max": 9.43847990129143e-05, "clip_ratio/high_mean": 9.43847990129143e-05, "clip_ratio/low_mean": 9.666985392363535e-05, "clip_ratio/low_min": 9.666985392363535e-05, "clip_ratio/region_mean": 0.00019105465293654965, "completions/clipped_ratio": 0.001953125, "completions/max_length": 77.0, "completions/max_terminated_length": 77.0, "completions/mean_length": 51.697265625, "completions/mean_terminated_length": 51.64775085449219, "completions/min_length": 32.0, "completions/min_terminated_length": 32.0, "entropy": 0.19397432108720145, "epoch": 0.03520625889046942, "frac_reward_zero_std": 0.59375, "grad_norm": 0.5314783453941345, "kl": 0.24874685290786955, "learning_rate": 4.162986219846037e-07, "loss": -0.0036, "num_tokens": 9515800.0, "reward": 1.185546875, "reward_std": 0.17774078249931335, "rewards/equation_reward_func/mean": 0.1875, "rewards/equation_reward_func/std": 0.39069411158561707, "rewards/format_reward_func/mean": 0.998046875, "rewards/format_reward_func/std": 0.04419417306780815, "sampling/importance_sampling_ratio/max": 1.5407081842422485, "sampling/importance_sampling_ratio/mean": 0.9988886117935181, "sampling/importance_sampling_ratio/min": 5.010540246597195e-11, "sampling/sampling_logp_difference/max": 23.71689224243164, "sampling/sampling_logp_difference/mean": 0.028378766030073166, "step": 132 }, { "clip_ratio/high_max": 0.00010051935613672767, "clip_ratio/high_mean": 0.00010051935613672767, "clip_ratio/low_mean": 0.00016403257419976094, "clip_ratio/low_min": 0.00016403257419976094, "clip_ratio/region_mean": 0.00026455193033648864, "completions/clipped_ratio": 0.0, "completions/max_length": 87.0, "completions/max_terminated_length": 87.0, "completions/mean_length": 52.8671875, "completions/mean_terminated_length": 52.8671875, "completions/min_length": 27.0, "completions/min_terminated_length": 27.0, "entropy": 0.19188633354173767, "epoch": 0.03573968705547653, "frac_reward_zero_std": 0.71875, "grad_norm": 0.542492687702179, "kl": 0.24353267459405792, "learning_rate": 4.135913860012219e-07, "loss": 0.0026, "num_tokens": 9620932.0, "reward": 1.10546875, "reward_std": 0.1255940943956375, "rewards/equation_reward_func/mean": 0.10546875, "rewards/equation_reward_func/std": 0.3074568510055542, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.3877536058425903, "sampling/importance_sampling_ratio/mean": 0.9999425411224365, "sampling/importance_sampling_ratio/min": 0.4762346148490906, "sampling/sampling_logp_difference/max": 0.741844654083252, "sampling/sampling_logp_difference/mean": 0.007330980617552996, "step": 134 }, { "clip_ratio/high_max": 3.425126649542815e-05, "clip_ratio/high_mean": 3.425126649542815e-05, "clip_ratio/low_mean": 9.994629848127563e-05, "clip_ratio/low_min": 9.994629848127563e-05, "clip_ratio/region_mean": 0.00013419756497670378, "completions/clipped_ratio": 0.0, "completions/max_length": 99.0, "completions/max_terminated_length": 99.0, "completions/mean_length": 52.87109375, "completions/mean_terminated_length": 52.87109375, "completions/min_length": 25.0, "completions/min_terminated_length": 25.0, "entropy": 0.18152287602424622, "epoch": 0.03627311522048364, "frac_reward_zero_std": 0.671875, "grad_norm": 0.7582495212554932, "kl": 0.5030062202778127, "learning_rate": 4.10850176566091e-07, "loss": -0.0083, "num_tokens": 9726154.0, "reward": 1.109375, "reward_std": 0.13704082369804382, "rewards/equation_reward_func/mean": 0.1171875, "rewards/equation_reward_func/std": 0.32195815443992615, "rewards/format_reward_func/mean": 0.9921875, "rewards/format_reward_func/std": 0.08812850713729858, "sampling/importance_sampling_ratio/max": 1.4538846015930176, "sampling/importance_sampling_ratio/mean": 1.0002654790878296, "sampling/importance_sampling_ratio/min": 0.39450007677078247, "sampling/sampling_logp_difference/max": 0.93013596534729, "sampling/sampling_logp_difference/mean": 0.0068960110656917095, "step": 136 }, { "clip_ratio/high_max": 0.00012915206025354564, "clip_ratio/high_mean": 0.00012915206025354564, "clip_ratio/low_mean": 0.00026444589861461683, "clip_ratio/low_min": 0.00026444589861461683, "clip_ratio/region_mean": 0.0003935979588681625, "completions/clipped_ratio": 0.0, "completions/max_length": 87.0, "completions/max_terminated_length": 87.0, "completions/mean_length": 53.25, "completions/mean_terminated_length": 53.25, "completions/min_length": 32.0, "completions/min_terminated_length": 32.0, "entropy": 0.17901599779725075, "epoch": 0.036806543385490754, "frac_reward_zero_std": 0.59375, "grad_norm": 0.7346581816673279, "kl": 0.20770272612571716, "learning_rate": 4.080755629533566e-07, "loss": 0.0198, "num_tokens": 9831426.0, "reward": 1.150390625, "reward_std": 0.1896313577890396, "rewards/equation_reward_func/mean": 0.16015625, "rewards/equation_reward_func/std": 0.3671095669269562, "rewards/format_reward_func/mean": 0.990234375, "rewards/format_reward_func/std": 0.09843364357948303, "sampling/importance_sampling_ratio/max": 1.4474687576293945, "sampling/importance_sampling_ratio/mean": 1.00001859664917, "sampling/importance_sampling_ratio/min": 0.6884425282478333, "sampling/sampling_logp_difference/max": 0.3733234405517578, "sampling/sampling_logp_difference/mean": 0.006761064287275076, "step": 138 }, { "clip_ratio/high_max": 0.00010025367050224708, "clip_ratio/high_mean": 0.00010025367050224708, "clip_ratio/low_mean": 0.0001902976784751647, "clip_ratio/low_min": 0.0001902976784751647, "clip_ratio/region_mean": 0.0002905513489774118, "completions/clipped_ratio": 0.0, "completions/max_length": 78.0, "completions/max_terminated_length": 78.0, "completions/mean_length": 53.2734375, "completions/mean_terminated_length": 53.2734375, "completions/min_length": 32.0, "completions/min_terminated_length": 32.0, "entropy": 0.18027027406626278, "epoch": 0.03733997155049787, "frac_reward_zero_std": 0.703125, "grad_norm": 0.530332624912262, "kl": 0.2300557821161217, "learning_rate": 4.052681213742971e-07, "loss": -0.012, "num_tokens": 9936646.0, "reward": 1.099609375, "reward_std": 0.11889472603797913, "rewards/equation_reward_func/mean": 0.103515625, "rewards/equation_reward_func/std": 0.30492907762527466, "rewards/format_reward_func/mean": 0.99609375, "rewards/format_reward_func/std": 0.06243881583213806, "sampling/importance_sampling_ratio/max": 1.6482826471328735, "sampling/importance_sampling_ratio/mean": 0.9998129606246948, "sampling/importance_sampling_ratio/min": 0.4797065258026123, "sampling/sampling_logp_difference/max": 0.7345807552337646, "sampling/sampling_logp_difference/mean": 0.006873577367514372, "step": 140 }, { "clip_ratio/high_max": 6.44635519064549e-05, "clip_ratio/high_mean": 6.44635519064549e-05, "clip_ratio/low_mean": 0.0001672606675937358, "clip_ratio/low_min": 0.0001672606675937358, "clip_ratio/region_mean": 0.0002317242195001907, "completions/clipped_ratio": 0.0, "completions/max_length": 77.0, "completions/max_terminated_length": 77.0, "completions/mean_length": 52.615234375, "completions/mean_terminated_length": 52.615234375, "completions/min_length": 31.0, "completions/min_terminated_length": 31.0, "entropy": 0.16615521907806396, "epoch": 0.03787339971550498, "frac_reward_zero_std": 0.6875, "grad_norm": 0.6144423484802246, "kl": 0.22150362903873125, "learning_rate": 4.024284348576611e-07, "loss": 0.0031, "num_tokens": 10041545.0, "reward": 1.12109375, "reward_std": 0.13269482553005219, "rewards/equation_reward_func/mean": 0.12109375, "rewards/equation_reward_func/std": 0.3265552520751953, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4515355825424194, "sampling/importance_sampling_ratio/mean": 0.9999923706054688, "sampling/importance_sampling_ratio/min": 0.5423057079315186, "sampling/sampling_logp_difference/max": 0.6119253635406494, "sampling/sampling_logp_difference/mean": 0.006486841011792421, "step": 142 }, { "clip_ratio/high_max": 3.2150206000854574e-05, "clip_ratio/high_mean": 3.2150206000854574e-05, "clip_ratio/low_mean": 0.0001318561650502185, "clip_ratio/low_min": 0.0001318561650502185, "clip_ratio/region_mean": 0.00016400637105107307, "completions/clipped_ratio": 0.0, "completions/max_length": 150.0, "completions/max_terminated_length": 150.0, "completions/mean_length": 54.080078125, "completions/mean_terminated_length": 54.080078125, "completions/min_length": 31.0, "completions/min_terminated_length": 31.0, "entropy": 0.18016350434886086, "epoch": 0.03840682788051209, "frac_reward_zero_std": 0.75, "grad_norm": 0.6359029412269592, "kl": 0.2206818246179157, "learning_rate": 3.9955709312858744e-07, "loss": -0.0016, "num_tokens": 10147442.0, "reward": 1.07421875, "reward_std": 0.1060032919049263, "rewards/equation_reward_func/mean": 0.07421875, "rewards/equation_reward_func/std": 0.2623828947544098, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.5840762853622437, "sampling/importance_sampling_ratio/mean": 0.9997929930686951, "sampling/importance_sampling_ratio/min": 0.47793424129486084, "sampling/sampling_logp_difference/max": 0.7382822036743164, "sampling/sampling_logp_difference/mean": 0.007104428485035896, "step": 144 }, { "clip_ratio/high_max": 6.125199272193843e-05, "clip_ratio/high_mean": 6.125199272193843e-05, "clip_ratio/low_mean": 6.607002837376462e-05, "clip_ratio/low_min": 6.607002837376462e-05, "clip_ratio/region_mean": 0.00012732202109570304, "completions/clipped_ratio": 0.0, "completions/max_length": 85.5, "completions/max_terminated_length": 85.5, "completions/mean_length": 54.5869140625, "completions/mean_terminated_length": 54.5869140625, "completions/min_length": 33.5, "completions/min_terminated_length": 33.5, "entropy": 0.1835891335374779, "epoch": 0.038940256045519206, "frac_reward_zero_std": 0.703125, "grad_norm": 0.6189127564430237, "kl": 0.2157546675039662, "learning_rate": 3.9665469248613616e-07, "loss": -0.0033, "num_tokens": 10359667.0, "reward": 1.1298828125, "reward_std": 0.1303960159420967, "rewards/equation_reward_func/mean": 0.1298828125, "rewards/equation_reward_func/std": 0.336248517036438, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.571405291557312, "sampling/importance_sampling_ratio/mean": 1.0000739693641663, "sampling/importance_sampling_ratio/min": 0.6091319620609283, "sampling/sampling_logp_difference/max": 0.49585986137390137, "sampling/sampling_logp_difference/mean": 0.007004189770668745, "step": 146 }, { "clip_ratio/high_max": 0.00012751224696532718, "clip_ratio/high_mean": 0.00012751224696532718, "clip_ratio/low_mean": 0.0001873280400306814, "clip_ratio/low_min": 0.0001873280400306814, "clip_ratio/region_mean": 0.0003148402869960086, "completions/clipped_ratio": 0.0, "completions/max_length": 143.0, "completions/max_terminated_length": 143.0, "completions/mean_length": 56.111328125, "completions/mean_terminated_length": 56.111328125, "completions/min_length": 32.0, "completions/min_terminated_length": 32.0, "entropy": 0.18567656063371235, "epoch": 0.039473684210526314, "frac_reward_zero_std": 0.734375, "grad_norm": 0.6717353463172913, "kl": 0.33979356123341453, "learning_rate": 3.9372183567945314e-07, "loss": 0.0086, "num_tokens": 10466764.0, "reward": 1.099609375, "reward_std": 0.12263225764036179, "rewards/equation_reward_func/mean": 0.099609375, "rewards/equation_reward_func/std": 0.29977133870124817, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.6148015260696411, "sampling/importance_sampling_ratio/mean": 1.0001806020736694, "sampling/importance_sampling_ratio/min": 0.5403767228126526, "sampling/sampling_logp_difference/max": 0.6154887676239014, "sampling/sampling_logp_difference/mean": 0.006991642992943525, "step": 148 }, { "clip_ratio/high_max": 0.00015601687805934084, "clip_ratio/high_mean": 0.00015601687805934084, "clip_ratio/low_mean": 9.35077195107523e-05, "clip_ratio/low_min": 9.35077195107523e-05, "clip_ratio/region_mean": 0.0002495245975700931, "completions/clipped_ratio": 0.0, "completions/max_length": 87.0, "completions/max_terminated_length": 87.0, "completions/mean_length": 54.890625, "completions/mean_terminated_length": 54.890625, "completions/min_length": 32.0, "completions/min_terminated_length": 32.0, "entropy": 0.18657905691199833, "epoch": 0.04000711237553343, "frac_reward_zero_std": 0.578125, "grad_norm": 0.7030015587806702, "kl": 0.21585925627085897, "learning_rate": 3.907591317825956e-07, "loss": -0.0152, "num_tokens": 10572988.0, "reward": 1.16796875, "reward_std": 0.18523895740509033, "rewards/equation_reward_func/mean": 0.169921875, "rewards/equation_reward_func/std": 0.3759314715862274, "rewards/format_reward_func/mean": 0.998046875, "rewards/format_reward_func/std": 0.04419417306780815, "sampling/importance_sampling_ratio/max": 1.6140424013137817, "sampling/importance_sampling_ratio/mean": 1.0000890493392944, "sampling/importance_sampling_ratio/min": 0.5526527166366577, "sampling/sampling_logp_difference/max": 0.5930254459381104, "sampling/sampling_logp_difference/mean": 0.007370211649686098, "step": 150 }, { "clip_ratio/high_max": 6.338281026627455e-05, "clip_ratio/high_mean": 6.338281026627455e-05, "clip_ratio/low_mean": 0.00012458148477081623, "clip_ratio/low_min": 0.00012458148477081623, "clip_ratio/region_mean": 0.00018796429503709078, "completions/clipped_ratio": 0.0, "completions/max_length": 85.0, "completions/max_terminated_length": 85.0, "completions/mean_length": 54.830078125, "completions/mean_terminated_length": 54.830078125, "completions/min_length": 33.0, "completions/min_terminated_length": 33.0, "entropy": 0.19338507536384794, "epoch": 0.04054054054054054, "frac_reward_zero_std": 0.703125, "grad_norm": 0.6828270554542542, "kl": 0.28864950748781365, "learning_rate": 3.877671960680443e-07, "loss": 0.016, "num_tokens": 10679221.0, "reward": 1.154296875, "reward_std": 0.12717056274414062, "rewards/equation_reward_func/mean": 0.154296875, "rewards/equation_reward_func/std": 0.36158639192581177, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4537763595581055, "sampling/importance_sampling_ratio/mean": 0.9997088313102722, "sampling/importance_sampling_ratio/min": 0.608879029750824, "sampling/sampling_logp_difference/max": 0.4961357116699219, "sampling/sampling_logp_difference/mean": 0.007639226969331503, "step": 152 }, { "clip_ratio/high_max": 0.0001902342935661889, "clip_ratio/high_mean": 0.0001902342935661889, "clip_ratio/low_mean": 0.00021917261053911515, "clip_ratio/low_min": 0.00021917261053911515, "clip_ratio/region_mean": 0.00040940690410530404, "completions/clipped_ratio": 0.0, "completions/max_length": 85.0, "completions/max_terminated_length": 85.0, "completions/mean_length": 54.658203125, "completions/mean_terminated_length": 54.658203125, "completions/min_length": 33.0, "completions/min_terminated_length": 33.0, "entropy": 0.18605521445473036, "epoch": 0.04107396870554765, "frac_reward_zero_std": 0.71875, "grad_norm": 0.5871764421463013, "kl": 0.2087138651145829, "learning_rate": 3.847466498789282e-07, "loss": -0.0096, "num_tokens": 10785270.0, "reward": 1.11328125, "reward_std": 0.12815652787685394, "rewards/equation_reward_func/mean": 0.11328125, "rewards/equation_reward_func/std": 0.3172462284564972, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4995094537734985, "sampling/importance_sampling_ratio/mean": 0.9998587369918823, "sampling/importance_sampling_ratio/min": 0.6065520644187927, "sampling/sampling_logp_difference/max": 0.49996471405029297, "sampling/sampling_logp_difference/mean": 0.007526198402047157, "step": 154 }, { "clip_ratio/high_max": 0.00012783622433845367, "clip_ratio/high_mean": 0.00012783622433845367, "clip_ratio/low_mean": 3.211303717560238e-05, "clip_ratio/low_min": 3.211303717560238e-05, "clip_ratio/region_mean": 0.00015994926151405607, "completions/clipped_ratio": 0.0, "completions/max_length": 87.0, "completions/max_terminated_length": 87.0, "completions/mean_length": 55.439453125, "completions/mean_terminated_length": 55.439453125, "completions/min_length": 36.0, "completions/min_terminated_length": 36.0, "entropy": 0.18379438585705227, "epoch": 0.041607396870554765, "frac_reward_zero_std": 0.71875, "grad_norm": 0.5001620054244995, "kl": 0.21861587092280388, "learning_rate": 3.816981204999882e-07, "loss": -0.0009, "num_tokens": 10891887.0, "reward": 1.119140625, "reward_std": 0.12671181559562683, "rewards/equation_reward_func/mean": 0.119140625, "rewards/equation_reward_func/std": 0.32427072525024414, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.8568929433822632, "sampling/importance_sampling_ratio/mean": 1.0003390312194824, "sampling/importance_sampling_ratio/min": 0.4759238362312317, "sampling/sampling_logp_difference/max": 0.742497444152832, "sampling/sampling_logp_difference/mean": 0.00747784785926342, "step": 156 }, { "clip_ratio/high_max": 0.00038483808101672266, "clip_ratio/high_mean": 0.00038483808101672266, "clip_ratio/low_mean": 0.00026095764931394823, "clip_ratio/low_min": 0.00026095764931394823, "clip_ratio/region_mean": 0.0006457957303306708, "completions/clipped_ratio": 0.0, "completions/max_length": 85.0, "completions/max_terminated_length": 85.0, "completions/mean_length": 53.755859375, "completions/mean_terminated_length": 53.755859375, "completions/min_length": 37.0, "completions/min_terminated_length": 37.0, "entropy": 0.1828179011742274, "epoch": 0.04214082503556188, "frac_reward_zero_std": 0.671875, "grad_norm": 0.5133856534957886, "kl": 0.3109063622024324, "learning_rate": 3.786222410273078e-07, "loss": -0.0017, "num_tokens": 10997298.0, "reward": 1.208984375, "reward_std": 0.16108018159866333, "rewards/equation_reward_func/mean": 0.2109375, "rewards/equation_reward_func/std": 0.4083731174468994, "rewards/format_reward_func/mean": 0.998046875, "rewards/format_reward_func/std": 0.04419417306780815, "sampling/importance_sampling_ratio/max": 1.445540189743042, "sampling/importance_sampling_ratio/mean": 0.9998924136161804, "sampling/importance_sampling_ratio/min": 0.4234831929206848, "sampling/sampling_logp_difference/max": 0.8592414855957031, "sampling/sampling_logp_difference/mean": 0.0071702562272548676, "step": 158 }, { "clip_ratio/high_max": 9.23872834795879e-05, "clip_ratio/high_mean": 9.23872834795879e-05, "clip_ratio/low_mean": 3.1494079545761146e-05, "clip_ratio/low_min": 3.1494079545761146e-05, "clip_ratio/region_mean": 0.00012388136302534904, "completions/clipped_ratio": 0.0, "completions/max_length": 83.0, "completions/max_terminated_length": 83.0, "completions/mean_length": 54.71875, "completions/mean_terminated_length": 54.71875, "completions/min_length": 41.0, "completions/min_terminated_length": 41.0, "entropy": 0.19189577591088083, "epoch": 0.04267425320056899, "frac_reward_zero_std": 0.78125, "grad_norm": 0.4394693672657013, "kl": 0.1994515239364571, "learning_rate": 3.755196502368361e-07, "loss": 0.0055, "num_tokens": 11103290.0, "reward": 1.140625, "reward_std": 0.09863612055778503, "rewards/equation_reward_func/mean": 0.140625, "rewards/equation_reward_func/std": 0.3479743003845215, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.6096035242080688, "sampling/importance_sampling_ratio/mean": 1.0000537633895874, "sampling/importance_sampling_ratio/min": 0.6076365113258362, "sampling/sampling_logp_difference/max": 0.49817848205566406, "sampling/sampling_logp_difference/mean": 0.00804964266717434, "step": 160 }, { "clip_ratio/high_max": 0.00012591962068755593, "clip_ratio/high_mean": 0.00012591962068755593, "clip_ratio/low_mean": 9.727939484744436e-05, "clip_ratio/low_min": 9.727939484744436e-05, "clip_ratio/region_mean": 0.0002231990155350003, "completions/clipped_ratio": 0.0, "completions/max_length": 98.5, "completions/max_terminated_length": 98.5, "completions/mean_length": 55.3720703125, "completions/mean_terminated_length": 55.3720703125, "completions/min_length": 33.0, "completions/min_terminated_length": 33.0, "entropy": 0.18139980029728678, "epoch": 0.0432076813655761, "frac_reward_zero_std": 0.625, "grad_norm": 0.4797929525375366, "kl": 0.20843237472905052, "learning_rate": 3.723909924517314e-07, "loss": 0.0063, "num_tokens": 11316079.0, "reward": 1.150390625, "reward_std": 0.1634674072265625, "rewards/equation_reward_func/mean": 0.1513671875, "rewards/equation_reward_func/std": 0.35236628353595734, "rewards/format_reward_func/mean": 0.9990234375, "rewards/format_reward_func/std": 0.022097086533904076, "sampling/importance_sampling_ratio/max": 1.4947065114974976, "sampling/importance_sampling_ratio/mean": 0.999882847070694, "sampling/importance_sampling_ratio/min": 0.47640350461006165, "sampling/sampling_logp_difference/max": 0.741499662399292, "sampling/sampling_logp_difference/mean": 0.007098018890246749, "step": 162 }, { "clip_ratio/high_max": 0.0001873802587700387, "clip_ratio/high_mean": 0.0001873802587700387, "clip_ratio/low_mean": 0.000373479687065507, "clip_ratio/low_min": 0.000373479687065507, "clip_ratio/region_mean": 0.0005608599458355457, "completions/clipped_ratio": 0.0, "completions/max_length": 79.0, "completions/max_terminated_length": 79.0, "completions/mean_length": 55.49609375, "completions/mean_terminated_length": 55.49609375, "completions/min_length": 43.0, "completions/min_terminated_length": 43.0, "entropy": 0.1870541183484925, "epoch": 0.043741109530583216, "frac_reward_zero_std": 0.703125, "grad_norm": 0.6516048908233643, "kl": 0.3076294929616981, "learning_rate": 3.692369174085534e-07, "loss": 0.001, "num_tokens": 11422629.0, "reward": 1.173828125, "reward_std": 0.13640210032463074, "rewards/equation_reward_func/mean": 0.17578125, "rewards/equation_reward_func/std": 0.3810062110424042, "rewards/format_reward_func/mean": 0.998046875, "rewards/format_reward_func/std": 0.04419417306780815, "sampling/importance_sampling_ratio/max": 1.6477184295654297, "sampling/importance_sampling_ratio/mean": 1.0000516176223755, "sampling/importance_sampling_ratio/min": 0.5377401113510132, "sampling/sampling_logp_difference/max": 0.6203799247741699, "sampling/sampling_logp_difference/mean": 0.0076889763586223125, "step": 164 }, { "clip_ratio/high_max": 0.00016062219704811773, "clip_ratio/high_mean": 0.00016062219704811773, "clip_ratio/low_mean": 6.016539151055945e-05, "clip_ratio/low_min": 6.016539151055945e-05, "clip_ratio/region_mean": 0.00022078758855867718, "completions/clipped_ratio": 0.0, "completions/max_length": 80.0, "completions/max_terminated_length": 80.0, "completions/mean_length": 54.98828125, "completions/mean_terminated_length": 54.98828125, "completions/min_length": 43.0, "completions/min_terminated_length": 43.0, "entropy": 0.18302446396814454, "epoch": 0.044274537695590324, "frac_reward_zero_std": 0.71875, "grad_norm": 0.6432575583457947, "kl": 0.24118708156877094, "learning_rate": 3.6605808012233004e-07, "loss": -0.0004, "num_tokens": 11528847.0, "reward": 1.1640625, "reward_std": 0.11363443732261658, "rewards/equation_reward_func/mean": 0.1640625, "rewards/equation_reward_func/std": 0.37069445848464966, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4534339904785156, "sampling/importance_sampling_ratio/mean": 0.9999691247940063, "sampling/importance_sampling_ratio/min": 0.6069241166114807, "sampling/sampling_logp_difference/max": 0.49935150146484375, "sampling/sampling_logp_difference/mean": 0.0071168760769069195, "step": 166 }, { "clip_ratio/high_max": 0.0001567405900762727, "clip_ratio/high_mean": 0.0001567405900762727, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0001567405900762727, "completions/clipped_ratio": 0.0, "completions/max_length": 86.0, "completions/max_terminated_length": 86.0, "completions/mean_length": 54.84375, "completions/mean_terminated_length": 54.84375, "completions/min_length": 32.0, "completions/min_terminated_length": 32.0, "entropy": 0.17809889341394106, "epoch": 0.04480796586059744, "frac_reward_zero_std": 0.6875, "grad_norm": 0.5978209972381592, "kl": 0.18232033277551332, "learning_rate": 3.628551407505292e-07, "loss": -0.0072, "num_tokens": 11634959.0, "reward": 1.16015625, "reward_std": 0.1343441903591156, "rewards/equation_reward_func/mean": 0.162109375, "rewards/equation_reward_func/std": 0.3689115643501282, "rewards/format_reward_func/mean": 0.998046875, "rewards/format_reward_func/std": 0.04419417306780815, "sampling/importance_sampling_ratio/max": 1.5012110471725464, "sampling/importance_sampling_ratio/mean": 1.0001106262207031, "sampling/importance_sampling_ratio/min": 0.5386102795600891, "sampling/sampling_logp_difference/max": 0.6187629699707031, "sampling/sampling_logp_difference/mean": 0.007097771856933832, "step": 168 }, { "clip_ratio/high_max": 3.233734363069137e-05, "clip_ratio/high_mean": 3.233734363069137e-05, "clip_ratio/low_mean": 0.00019210456068524055, "clip_ratio/low_min": 0.00019210456068524055, "clip_ratio/region_mean": 0.00022444190431593193, "completions/clipped_ratio": 0.0, "completions/max_length": 89.0, "completions/max_terminated_length": 89.0, "completions/mean_length": 54.818359375, "completions/mean_terminated_length": 54.818359375, "completions/min_length": 42.0, "completions/min_terminated_length": 42.0, "entropy": 0.1777913181318177, "epoch": 0.04534139402560455, "frac_reward_zero_std": 0.6875, "grad_norm": 0.5563479661941528, "kl": 0.2434103993905915, "learning_rate": 3.5962876445596224e-07, "loss": 0.0068, "num_tokens": 11741194.0, "reward": 1.10546875, "reward_std": 0.13098736107349396, "rewards/equation_reward_func/mean": 0.10546875, "rewards/equation_reward_func/std": 0.3074568510055542, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.5672227144241333, "sampling/importance_sampling_ratio/mean": 0.9999104142189026, "sampling/importance_sampling_ratio/min": 0.5404222011566162, "sampling/sampling_logp_difference/max": 0.6154046058654785, "sampling/sampling_logp_difference/mean": 0.007332408335059881, "step": 170 }, { "clip_ratio/high_max": 0.00015626028956224522, "clip_ratio/high_mean": 0.00015626028956224522, "clip_ratio/low_mean": 0.0001587454591774278, "clip_ratio/low_min": 0.0001587454591774278, "clip_ratio/region_mean": 0.00031500574873967306, "completions/clipped_ratio": 0.0, "completions/max_length": 87.0, "completions/max_terminated_length": 87.0, "completions/mean_length": 53.61328125, "completions/mean_terminated_length": 53.61328125, "completions/min_length": 32.0, "completions/min_terminated_length": 32.0, "entropy": 0.1705088723036978, "epoch": 0.04587482219061166, "frac_reward_zero_std": 0.65625, "grad_norm": 0.6292311549186707, "kl": 0.20615031073490778, "learning_rate": 3.563796212686475e-07, "loss": -0.004, "num_tokens": 11846532.0, "reward": 1.142578125, "reward_std": 0.14768996834754944, "rewards/equation_reward_func/mean": 0.142578125, "rewards/equation_reward_func/std": 0.3499840497970581, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.6039351224899292, "sampling/importance_sampling_ratio/mean": 1.0001249313354492, "sampling/importance_sampling_ratio/min": 0.6371393799781799, "sampling/sampling_logp_difference/max": 0.4724600315093994, "sampling/sampling_logp_difference/mean": 0.0072227707132697105, "step": 172 }, { "clip_ratio/high_max": 9.414075045949883e-05, "clip_ratio/high_mean": 9.414075045949883e-05, "clip_ratio/low_mean": 0.0002569324860814959, "clip_ratio/low_min": 0.0002569324860814959, "clip_ratio/region_mean": 0.0003510732365409947, "completions/clipped_ratio": 0.0, "completions/max_length": 78.0, "completions/max_terminated_length": 78.0, "completions/mean_length": 54.255859375, "completions/mean_terminated_length": 54.255859375, "completions/min_length": 43.0, "completions/min_terminated_length": 43.0, "entropy": 0.17187187944849333, "epoch": 0.046408250355618776, "frac_reward_zero_std": 0.6875, "grad_norm": 0.5958592891693115, "kl": 0.22016057992974916, "learning_rate": 3.531083859466635e-07, "loss": 0.0044, "num_tokens": 11952343.0, "reward": 1.14453125, "reward_std": 0.1269814372062683, "rewards/equation_reward_func/mean": 0.14453125, "rewards/equation_reward_func/std": 0.35197147727012634, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.427687168121338, "sampling/importance_sampling_ratio/mean": 1.0000633001327515, "sampling/importance_sampling_ratio/min": 0.609822154045105, "sampling/sampling_logp_difference/max": 0.49458789825439453, "sampling/sampling_logp_difference/mean": 0.007299026474356651, "step": 174 }, { "clip_ratio/high_max": 0.0001271874352823943, "clip_ratio/high_mean": 0.0001271874352823943, "clip_ratio/low_mean": 0.00016112958676078252, "clip_ratio/low_min": 0.00016112958676078252, "clip_ratio/region_mean": 0.0002883170220431768, "completions/clipped_ratio": 0.0, "completions/max_length": 85.0, "completions/max_terminated_length": 85.0, "completions/mean_length": 54.880859375, "completions/mean_terminated_length": 54.880859375, "completions/min_length": 35.0, "completions/min_terminated_length": 35.0, "entropy": 0.1630988936457369, "epoch": 0.04694167852062589, "frac_reward_zero_std": 0.71875, "grad_norm": 0.5851055383682251, "kl": 0.4657417970399062, "learning_rate": 3.498157378360204e-07, "loss": -0.0006, "num_tokens": 12058658.0, "reward": 1.119140625, "reward_std": 0.11849410086870193, "rewards/equation_reward_func/mean": 0.12109375, "rewards/equation_reward_func/std": 0.3265552520751953, "rewards/format_reward_func/mean": 0.998046875, "rewards/format_reward_func/std": 0.04419417306780815, "sampling/importance_sampling_ratio/max": 1.4064489603042603, "sampling/importance_sampling_ratio/mean": 1.0000070333480835, "sampling/importance_sampling_ratio/min": 0.5667109489440918, "sampling/sampling_logp_difference/max": 0.5679059028625488, "sampling/sampling_logp_difference/mean": 0.0071370163932442665, "step": 176 }, { "clip_ratio/high_max": 2.9932950080061953e-05, "clip_ratio/high_mean": 2.9932950080061953e-05, "clip_ratio/low_mean": 3.128128203873833e-05, "clip_ratio/low_min": 3.128128203873833e-05, "clip_ratio/region_mean": 6.121423211880028e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 79.5, "completions/max_terminated_length": 79.5, "completions/mean_length": 54.720703125, "completions/mean_terminated_length": 54.720703125, "completions/min_length": 34.5, "completions/min_terminated_length": 34.5, "entropy": 0.1649447451863024, "epoch": 0.047475106685633005, "frac_reward_zero_std": 0.75, "grad_norm": 0.5909639000892639, "kl": 0.45460431194967693, "learning_rate": 3.465023607295784e-07, "loss": 0.0043, "num_tokens": 12270948.0, "reward": 1.107421875, "reward_std": 0.10856572538614273, "rewards/equation_reward_func/mean": 0.107421875, "rewards/equation_reward_func/std": 0.3079848438501358, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.519451916217804, "sampling/importance_sampling_ratio/mean": 1.0000729262828827, "sampling/importance_sampling_ratio/min": 0.4784931391477585, "sampling/sampling_logp_difference/max": 0.745232105255127, "sampling/sampling_logp_difference/mean": 0.0071322976145893335, "step": 178 }, { "clip_ratio/high_max": 6.297189651781486e-05, "clip_ratio/high_mean": 6.297189651781486e-05, "clip_ratio/low_mean": 3.229974148174127e-05, "clip_ratio/low_min": 3.229974148174127e-05, "clip_ratio/region_mean": 9.527163799955613e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 103.0, "completions/max_terminated_length": 103.0, "completions/mean_length": 54.529296875, "completions/mean_terminated_length": 54.529296875, "completions/min_length": 41.0, "completions/min_terminated_length": 41.0, "entropy": 0.15816421620547771, "epoch": 0.04800853485064011, "frac_reward_zero_std": 0.703125, "grad_norm": 0.6006011962890625, "kl": 0.2346639575229751, "learning_rate": 3.4316894272504225e-07, "loss": -0.0081, "num_tokens": 12377027.0, "reward": 1.130859375, "reward_std": 0.12802553176879883, "rewards/equation_reward_func/mean": 0.130859375, "rewards/equation_reward_func/std": 0.33757632970809937, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.8673962354660034, "sampling/importance_sampling_ratio/mean": 0.9999069571495056, "sampling/importance_sampling_ratio/min": 0.4734019935131073, "sampling/sampling_logp_difference/max": 0.7478103637695312, "sampling/sampling_logp_difference/mean": 0.007345444522798061, "step": 180 }, { "clip_ratio/high_max": 6.469187145638798e-05, "clip_ratio/high_mean": 6.469187145638798e-05, "clip_ratio/low_mean": 0.0001901625332215594, "clip_ratio/low_min": 0.0001901625332215594, "clip_ratio/region_mean": 0.00025485440467794734, "completions/clipped_ratio": 0.0, "completions/max_length": 80.0, "completions/max_terminated_length": 80.0, "completions/mean_length": 54.880859375, "completions/mean_terminated_length": 54.880859375, "completions/min_length": 38.0, "completions/min_terminated_length": 38.0, "entropy": 0.15701940614316198, "epoch": 0.04854196301564723, "frac_reward_zero_std": 0.796875, "grad_norm": 0.620672881603241, "kl": 0.24056435086660916, "learning_rate": 3.398161760820628e-07, "loss": 0.0146, "num_tokens": 12483326.0, "reward": 1.078125, "reward_std": 0.08633885532617569, "rewards/equation_reward_func/mean": 0.078125, "rewards/equation_reward_func/std": 0.26863065361976624, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4245080947875977, "sampling/importance_sampling_ratio/mean": 1.0000905990600586, "sampling/importance_sampling_ratio/min": 0.7031283974647522, "sampling/sampling_logp_difference/max": 0.35382652282714844, "sampling/sampling_logp_difference/mean": 0.007266565226018429, "step": 182 }, { "clip_ratio/high_max": 0.00012663751517215537, "clip_ratio/high_mean": 0.00012663751517215537, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00012663751517215537, "completions/clipped_ratio": 0.0, "completions/max_length": 83.0, "completions/max_terminated_length": 83.0, "completions/mean_length": 54.255859375, "completions/mean_terminated_length": 54.255859375, "completions/min_length": 37.0, "completions/min_terminated_length": 37.0, "entropy": 0.16044065687391493, "epoch": 0.04907539118065434, "frac_reward_zero_std": 0.65625, "grad_norm": 0.6004108786582947, "kl": 0.18950829137530592, "learning_rate": 3.364447570784731e-07, "loss": -0.0007, "num_tokens": 12589121.0, "reward": 1.103515625, "reward_std": 0.14854492247104645, "rewards/equation_reward_func/mean": 0.10546875, "rewards/equation_reward_func/std": 0.3074568510055542, "rewards/format_reward_func/mean": 0.998046875, "rewards/format_reward_func/std": 0.04419417306780815, "sampling/importance_sampling_ratio/max": 1.444663643836975, "sampling/importance_sampling_ratio/mean": 1.00014328956604, "sampling/importance_sampling_ratio/min": 0.6588608026504517, "sampling/sampling_logp_difference/max": 0.41724300384521484, "sampling/sampling_logp_difference/mean": 0.007076562847942114, "step": 184 }, { "clip_ratio/high_max": 0.0002561409103994568, "clip_ratio/high_mean": 0.0002561409103994568, "clip_ratio/low_mean": 0.0002564327636112769, "clip_ratio/low_min": 0.0002564327636112769, "clip_ratio/region_mean": 0.0005125736740107337, "completions/clipped_ratio": 0.0, "completions/max_length": 78.0, "completions/max_terminated_length": 78.0, "completions/mean_length": 54.208984375, "completions/mean_terminated_length": 54.208984375, "completions/min_length": 38.0, "completions/min_terminated_length": 38.0, "entropy": 0.1514270328399208, "epoch": 0.04960881934566145, "frac_reward_zero_std": 0.65625, "grad_norm": 0.45418864488601685, "kl": 0.20794780593779352, "learning_rate": 3.3305538586569116e-07, "loss": 0.0119, "num_tokens": 12694964.0, "reward": 1.18359375, "reward_std": 0.15478995442390442, "rewards/equation_reward_func/mean": 0.18359375, "rewards/equation_reward_func/std": 0.3875311613082886, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.6467537879943848, "sampling/importance_sampling_ratio/mean": 0.9998853802680969, "sampling/importance_sampling_ratio/min": 0.48188647627830505, "sampling/sampling_logp_difference/max": 0.7300467491149902, "sampling/sampling_logp_difference/mean": 0.006952677387744188, "step": 186 }, { "clip_ratio/high_max": 0.0002880274227613376, "clip_ratio/high_mean": 0.0002880274227613376, "clip_ratio/low_mean": 0.00047546141043615836, "clip_ratio/low_min": 0.00047546141043615836, "clip_ratio/region_mean": 0.000763488836431255, "completions/clipped_ratio": 0.0, "completions/max_length": 79.0, "completions/max_terminated_length": 79.0, "completions/mean_length": 54.818359375, "completions/mean_terminated_length": 54.818359375, "completions/min_length": 38.0, "completions/min_terminated_length": 38.0, "entropy": 0.15331672401063973, "epoch": 0.050142247510668564, "frac_reward_zero_std": 0.640625, "grad_norm": 0.675754725933075, "kl": 0.22493650681442684, "learning_rate": 3.296487663233168e-07, "loss": -0.019, "num_tokens": 12801319.0, "reward": 1.14453125, "reward_std": 0.15150675177574158, "rewards/equation_reward_func/mean": 0.1484375, "rewards/equation_reward_func/std": 0.35588082671165466, "rewards/format_reward_func/mean": 0.99609375, "rewards/format_reward_func/std": 0.06243881583213806, "sampling/importance_sampling_ratio/max": 1.644798994064331, "sampling/importance_sampling_ratio/mean": 0.9999932646751404, "sampling/importance_sampling_ratio/min": 0.3253546357154846, "sampling/sampling_logp_difference/max": 1.1228394508361816, "sampling/sampling_logp_difference/mean": 0.007331944070756435, "step": 188 }, { "clip_ratio/high_max": 0.0001251859066542238, "clip_ratio/high_mean": 0.0001251859066542238, "clip_ratio/low_mean": 0.0002512690197262499, "clip_ratio/low_min": 0.0002512690197262499, "clip_ratio/region_mean": 0.0003764549263804737, "completions/clipped_ratio": 0.0, "completions/max_length": 109.0, "completions/max_terminated_length": 109.0, "completions/mean_length": 53.7421875, "completions/mean_terminated_length": 53.7421875, "completions/min_length": 35.0, "completions/min_terminated_length": 35.0, "entropy": 0.1532320384350088, "epoch": 0.05067567567567568, "frac_reward_zero_std": 0.65625, "grad_norm": 0.808289647102356, "kl": 0.21676196778813997, "learning_rate": 3.2622560591295606e-07, "loss": 0.009, "num_tokens": 12906755.0, "reward": 1.154296875, "reward_std": 0.1503761112689972, "rewards/equation_reward_func/mean": 0.16015625, "rewards/equation_reward_func/std": 0.3671095669269562, "rewards/format_reward_func/mean": 0.994140625, "rewards/format_reward_func/std": 0.07639661431312561, "sampling/importance_sampling_ratio/max": 1.5660736560821533, "sampling/importance_sampling_ratio/mean": 1.0004606246948242, "sampling/importance_sampling_ratio/min": 0.3685874342918396, "sampling/sampling_logp_difference/max": 0.998077392578125, "sampling/sampling_logp_difference/mean": 0.0070484536699950695, "step": 190 }, { "clip_ratio/high_max": 0.00012767854301879802, "clip_ratio/high_mean": 0.00012767854301879802, "clip_ratio/low_mean": 0.0001271537104104128, "clip_ratio/low_min": 0.0001271537104104128, "clip_ratio/region_mean": 0.00025483225342921086, "completions/clipped_ratio": 0.0, "completions/max_length": 88.0, "completions/max_terminated_length": 88.0, "completions/mean_length": 54.3984375, "completions/mean_terminated_length": 54.3984375, "completions/min_length": 33.0, "completions/min_terminated_length": 33.0, "entropy": 0.1483436483475897, "epoch": 0.051209103840682786, "frac_reward_zero_std": 0.6875, "grad_norm": 0.5794490575790405, "kl": 0.2033289468122853, "learning_rate": 3.227866155313002e-07, "loss": -0.0003, "num_tokens": 13012791.0, "reward": 1.15234375, "reward_std": 0.13749760389328003, "rewards/equation_reward_func/mean": 0.15234375, "rewards/equation_reward_func/std": 0.35970520973205566, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.3876280784606934, "sampling/importance_sampling_ratio/mean": 0.9999657869338989, "sampling/importance_sampling_ratio/min": 0.644438624382019, "sampling/sampling_logp_difference/max": 0.439375638961792, "sampling/sampling_logp_difference/mean": 0.0071494379080832005, "step": 192 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00019094683294598426, "clip_ratio/low_min": 0.00019094683294598426, "clip_ratio/region_mean": 0.00019094683294598426, "completions/clipped_ratio": 0.0, "completions/max_length": 86.0, "completions/max_terminated_length": 86.0, "completions/mean_length": 54.8193359375, "completions/mean_terminated_length": 54.8193359375, "completions/min_length": 40.5, "completions/min_terminated_length": 40.5, "entropy": 0.1460409445895089, "epoch": 0.0517425320056899, "frac_reward_zero_std": 0.6953125, "grad_norm": 0.5141434669494629, "kl": 0.20667694177892473, "learning_rate": 3.1933250936249213e-07, "loss": -0.0011, "num_tokens": 13225486.0, "reward": 1.1396484375, "reward_std": 0.1303311251103878, "rewards/equation_reward_func/mean": 0.140625, "rewards/equation_reward_func/std": 0.3479743003845215, "rewards/format_reward_func/mean": 0.9990234375, "rewards/format_reward_func/std": 0.022097086533904076, "sampling/importance_sampling_ratio/max": 1.4357757568359375, "sampling/importance_sampling_ratio/mean": 1.0000481605529785, "sampling/importance_sampling_ratio/min": 0.6091239154338837, "sampling/sampling_logp_difference/max": 0.5029124021530151, "sampling/sampling_logp_difference/mean": 0.007287219632416964, "step": 194 }, { "clip_ratio/high_max": 0.000539941748785269, "clip_ratio/high_mean": 0.000539941748785269, "clip_ratio/low_mean": 0.00046990027314879827, "clip_ratio/low_min": 0.00046990027314879827, "clip_ratio/region_mean": 0.0010098420154665494, "completions/clipped_ratio": 0.0, "completions/max_length": 81.0, "completions/max_terminated_length": 81.0, "completions/mean_length": 53.841796875, "completions/mean_terminated_length": 53.841796875, "completions/min_length": 36.0, "completions/min_terminated_length": 36.0, "entropy": 0.14552338793873787, "epoch": 0.052275960170697015, "frac_reward_zero_std": 0.609375, "grad_norm": 1.226148247718811, "kl": 0.2773982307149304, "learning_rate": 3.158640047298098e-07, "loss": 0.0025, "num_tokens": 13331053.0, "reward": 1.1875, "reward_std": 0.15940433740615845, "rewards/equation_reward_func/mean": 0.1875, "rewards/equation_reward_func/std": 0.39069411158561707, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4396930932998657, "sampling/importance_sampling_ratio/mean": 1.000059962272644, "sampling/importance_sampling_ratio/min": 0.621496319770813, "sampling/sampling_logp_difference/max": 0.47562527656555176, "sampling/sampling_logp_difference/mean": 0.006564574781805277, "step": 196 }, { "clip_ratio/high_max": 0.0003576159328480975, "clip_ratio/high_mean": 0.0003576159328480975, "clip_ratio/low_mean": 0.000351945691767873, "clip_ratio/low_min": 0.000351945691767873, "clip_ratio/region_mean": 0.0007095616246159705, "completions/clipped_ratio": 0.0, "completions/max_length": 78.0, "completions/max_terminated_length": 78.0, "completions/mean_length": 53.451171875, "completions/mean_terminated_length": 53.451171875, "completions/min_length": 32.0, "completions/min_terminated_length": 32.0, "entropy": 0.14037801759938398, "epoch": 0.05280938833570412, "frac_reward_zero_std": 0.75, "grad_norm": 0.4710004925727844, "kl": 0.24111700699561173, "learning_rate": 3.123818219466981e-07, "loss": 0.0009, "num_tokens": 13436436.0, "reward": 1.212890625, "reward_std": 0.1142803430557251, "rewards/equation_reward_func/mean": 0.212890625, "rewards/equation_reward_func/std": 0.409751296043396, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4310200214385986, "sampling/importance_sampling_ratio/mean": 1.0001682043075562, "sampling/importance_sampling_ratio/min": 0.6036964058876038, "sampling/sampling_logp_difference/max": 0.5046838521957397, "sampling/sampling_logp_difference/mean": 0.00685081584379077, "step": 198 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0001622872304223064, "clip_ratio/low_min": 0.0001622872304223064, "clip_ratio/region_mean": 0.0001622872304223064, "completions/clipped_ratio": 0.0, "completions/max_length": 74.0, "completions/max_terminated_length": 74.0, "completions/mean_length": 53.625, "completions/mean_terminated_length": 53.625, "completions/min_length": 38.0, "completions/min_terminated_length": 38.0, "entropy": 0.13289273199107912, "epoch": 0.05334281650071124, "frac_reward_zero_std": 0.703125, "grad_norm": 0.6205074787139893, "kl": 0.3644645079556439, "learning_rate": 3.088866841671789e-07, "loss": 0.0018, "num_tokens": 13541980.0, "reward": 1.146484375, "reward_std": 0.13825923204421997, "rewards/equation_reward_func/mean": 0.1484375, "rewards/equation_reward_func/std": 0.35588082671165466, "rewards/format_reward_func/mean": 0.998046875, "rewards/format_reward_func/std": 0.04419417306780815, "sampling/importance_sampling_ratio/max": 1.4064486026763916, "sampling/importance_sampling_ratio/mean": 1.0000766515731812, "sampling/importance_sampling_ratio/min": 0.607798159122467, "sampling/sampling_logp_difference/max": 0.4979124069213867, "sampling/sampling_logp_difference/mean": 0.006802752614021301, "step": 200 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.000385660015227687, "clip_ratio/low_min": 0.000385660015227687, "clip_ratio/region_mean": 0.000385660015227687, "completions/clipped_ratio": 0.0, "completions/max_length": 79.0, "completions/max_terminated_length": 79.0, "completions/mean_length": 54.3125, "completions/mean_terminated_length": 54.3125, "completions/min_length": 35.0, "completions/min_terminated_length": 35.0, "entropy": 0.13150467558039558, "epoch": 0.05387624466571835, "frac_reward_zero_std": 0.75, "grad_norm": 0.44971543550491333, "kl": 0.2712813439882464, "learning_rate": 3.0537931723567253e-07, "loss": -0.002, "num_tokens": 13648084.0, "reward": 1.140625, "reward_std": 0.10882970690727234, "rewards/equation_reward_func/mean": 0.140625, "rewards/equation_reward_func/std": 0.3479743003845215, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.5713891983032227, "sampling/importance_sampling_ratio/mean": 0.999980628490448, "sampling/importance_sampling_ratio/min": 0.5369875431060791, "sampling/sampling_logp_difference/max": 0.6217803955078125, "sampling/sampling_logp_difference/mean": 0.006598425097763538, "step": 202 }, { "clip_ratio/high_max": 0.00022619308502827253, "clip_ratio/high_mean": 0.00022619308502827253, "clip_ratio/low_mean": 0.000285603701033526, "clip_ratio/low_min": 0.000285603701033526, "clip_ratio/region_mean": 0.0005117967860617986, "completions/clipped_ratio": 0.0, "completions/max_length": 77.0, "completions/max_terminated_length": 77.0, "completions/mean_length": 54.66796875, "completions/mean_terminated_length": 54.66796875, "completions/min_length": 38.0, "completions/min_terminated_length": 38.0, "entropy": 0.12755409379800162, "epoch": 0.05440967283072546, "frac_reward_zero_std": 0.71875, "grad_norm": 0.6239610910415649, "kl": 0.2788337940971057, "learning_rate": 3.01860449536259e-07, "loss": 0.0013, "num_tokens": 13754410.0, "reward": 1.1015625, "reward_std": 0.12901148200035095, "rewards/equation_reward_func/mean": 0.1015625, "rewards/equation_reward_func/std": 0.30236753821372986, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.8019253015518188, "sampling/importance_sampling_ratio/mean": 1.0001003742218018, "sampling/importance_sampling_ratio/min": 0.48298075795173645, "sampling/sampling_logp_difference/max": 0.727778434753418, "sampling/sampling_logp_difference/mean": 0.0065957228653132915, "step": 204 }, { "clip_ratio/high_max": 0.00016267979258878363, "clip_ratio/high_mean": 0.00016267979258878363, "clip_ratio/low_mean": 0.0004494503470292936, "clip_ratio/low_min": 0.0004494503470292936, "clip_ratio/region_mean": 0.0006121301396180772, "completions/clipped_ratio": 0.0, "completions/max_length": 72.0, "completions/max_terminated_length": 72.0, "completions/mean_length": 52.990234375, "completions/mean_terminated_length": 52.990234375, "completions/min_length": 38.0, "completions/min_terminated_length": 38.0, "entropy": 0.12572518384291065, "epoch": 0.054943100995732574, "frac_reward_zero_std": 0.65625, "grad_norm": 0.7820398211479187, "kl": 0.22174248699512747, "learning_rate": 2.983308118414131e-07, "loss": 0.0092, "num_tokens": 13859429.0, "reward": 1.173828125, "reward_std": 0.14716199040412903, "rewards/equation_reward_func/mean": 0.17578125, "rewards/equation_reward_func/std": 0.3810062110424042, "rewards/format_reward_func/mean": 0.998046875, "rewards/format_reward_func/std": 0.04419417306780815, "sampling/importance_sampling_ratio/max": 1.473716378211975, "sampling/importance_sampling_ratio/mean": 1.0001775026321411, "sampling/importance_sampling_ratio/min": 0.6095648407936096, "sampling/sampling_logp_difference/max": 0.4950098991394043, "sampling/sampling_logp_difference/mean": 0.006203033495694399, "step": 206 }, { "clip_ratio/high_max": 0.00019132617227215733, "clip_ratio/high_mean": 0.00019132617227215733, "clip_ratio/low_mean": 0.00019700702978298068, "clip_ratio/low_min": 0.00019700702978298068, "clip_ratio/region_mean": 0.000388333202055138, "completions/clipped_ratio": 0.001953125, "completions/max_length": 75.0, "completions/max_terminated_length": 75.0, "completions/mean_length": 53.828125, "completions/mean_terminated_length": 53.78669357299805, "completions/min_length": 37.0, "completions/min_terminated_length": 37.0, "entropy": 0.12475637284417947, "epoch": 0.05547652916073969, "frac_reward_zero_std": 0.6875, "grad_norm": 0.7352174520492554, "kl": 0.19599058396286434, "learning_rate": 2.9479113716024275e-07, "loss": -0.0059, "num_tokens": 13965125.0, "reward": 1.14453125, "reward_std": 0.13664263486862183, "rewards/equation_reward_func/mean": 0.146484375, "rewards/equation_reward_func/std": 0.35393697023391724, "rewards/format_reward_func/mean": 0.998046875, "rewards/format_reward_func/std": 0.04419417306780815, "sampling/importance_sampling_ratio/max": 1.646515965461731, "sampling/importance_sampling_ratio/mean": 0.9995404481887817, "sampling/importance_sampling_ratio/min": 9.77650252420581e-11, "sampling/sampling_logp_difference/max": 23.04845428466797, "sampling/sampling_logp_difference/mean": 0.01847986876964569, "step": 208 }, { "clip_ratio/high_max": 3.379291673708293e-05, "clip_ratio/high_mean": 3.379291673708293e-05, "clip_ratio/low_mean": 0.00019734271983098652, "clip_ratio/low_min": 0.00019734271983098652, "clip_ratio/region_mean": 0.00023113563656806946, "completions/clipped_ratio": 0.0, "completions/max_length": 74.0, "completions/max_terminated_length": 74.0, "completions/mean_length": 53.04296875, "completions/mean_terminated_length": 53.04296875, "completions/min_length": 35.0, "completions/min_terminated_length": 35.0, "entropy": 0.12059136997494432, "epoch": 0.0560099573257468, "frac_reward_zero_std": 0.7265625, "grad_norm": 0.474179208278656, "kl": 0.19687053250769773, "learning_rate": 2.912421605862632e-07, "loss": 0.0008, "num_tokens": 14175505.0, "reward": 1.1142578125, "reward_std": 0.11813334748148918, "rewards/equation_reward_func/mean": 0.1142578125, "rewards/equation_reward_func/std": 0.31834255158901215, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4857916235923767, "sampling/importance_sampling_ratio/mean": 0.9999574422836304, "sampling/importance_sampling_ratio/min": 0.49108970165252686, "sampling/sampling_logp_difference/max": 0.7114477157592773, "sampling/sampling_logp_difference/mean": 0.006181263132020831, "step": 210 }, { "clip_ratio/high_max": 3.245067394648989e-05, "clip_ratio/high_mean": 3.245067394648989e-05, "clip_ratio/low_mean": 0.00036193533580646745, "clip_ratio/low_min": 0.00036193533580646745, "clip_ratio/region_mean": 0.0003943860097529574, "completions/clipped_ratio": 0.0, "completions/max_length": 72.0, "completions/max_terminated_length": 72.0, "completions/mean_length": 53.7109375, "completions/mean_terminated_length": 53.7109375, "completions/min_length": 36.0, "completions/min_terminated_length": 36.0, "entropy": 0.12079384653932518, "epoch": 0.05654338549075391, "frac_reward_zero_std": 0.6875, "grad_norm": 0.6931771039962769, "kl": 0.210977077898052, "learning_rate": 2.8768461914473794e-07, "loss": 0.0021, "num_tokens": 14281181.0, "reward": 1.205078125, "reward_std": 0.143476203083992, "rewards/equation_reward_func/mean": 0.205078125, "rewards/equation_reward_func/std": 0.4041535556316376, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.8660997152328491, "sampling/importance_sampling_ratio/mean": 1.0000596046447754, "sampling/importance_sampling_ratio/min": 0.6091339588165283, "sampling/sampling_logp_difference/max": 0.6238505840301514, "sampling/sampling_logp_difference/mean": 0.005528800189495087, "step": 212 }, { "clip_ratio/high_max": 0.00012854254843356708, "clip_ratio/high_mean": 0.00012854254843356708, "clip_ratio/low_mean": 0.00019423514863269197, "clip_ratio/low_min": 0.00019423514863269197, "clip_ratio/region_mean": 0.00032277769706625905, "completions/clipped_ratio": 0.0, "completions/max_length": 71.0, "completions/max_terminated_length": 71.0, "completions/mean_length": 53.357421875, "completions/mean_terminated_length": 53.357421875, "completions/min_length": 36.0, "completions/min_terminated_length": 36.0, "entropy": 0.11314595283733474, "epoch": 0.057076813655761026, "frac_reward_zero_std": 0.609375, "grad_norm": 0.5694147348403931, "kl": 0.28766577939192456, "learning_rate": 2.8411925163961926e-07, "loss": -0.0096, "num_tokens": 14386692.0, "reward": 1.146484375, "reward_std": 0.1631450653076172, "rewards/equation_reward_func/mean": 0.150390625, "rewards/equation_reward_func/std": 0.35780346393585205, "rewards/format_reward_func/mean": 0.99609375, "rewards/format_reward_func/std": 0.06243881583213806, "sampling/importance_sampling_ratio/max": 1.6400882005691528, "sampling/importance_sampling_ratio/mean": 1.0001121759414673, "sampling/importance_sampling_ratio/min": 0.6106104254722595, "sampling/sampling_logp_difference/max": 0.4947500228881836, "sampling/sampling_logp_difference/mean": 0.005793167278170586, "step": 214 }, { "clip_ratio/high_max": 9.60305559500638e-05, "clip_ratio/high_mean": 9.60305559500638e-05, "clip_ratio/low_mean": 0.0002618474051511536, "clip_ratio/low_min": 0.0002618474051511536, "clip_ratio/region_mean": 0.0003578779611012174, "completions/clipped_ratio": 0.0, "completions/max_length": 72.0, "completions/max_terminated_length": 72.0, "completions/mean_length": 52.5625, "completions/mean_terminated_length": 52.5625, "completions/min_length": 34.0, "completions/min_terminated_length": 34.0, "entropy": 0.11545825170146094, "epoch": 0.057610241820768134, "frac_reward_zero_std": 0.703125, "grad_norm": 0.8972283005714417, "kl": 0.4845621803154548, "learning_rate": 2.8054679850011825e-07, "loss": 0.0024, "num_tokens": 14491556.0, "reward": 1.1328125, "reward_std": 0.1345345377922058, "rewards/equation_reward_func/mean": 0.13671875, "rewards/equation_reward_func/std": 0.3438861668109894, "rewards/format_reward_func/mean": 0.99609375, "rewards/format_reward_func/std": 0.06243881583213806, "sampling/importance_sampling_ratio/max": 1.590267539024353, "sampling/importance_sampling_ratio/mean": 1.0001193284988403, "sampling/importance_sampling_ratio/min": 0.5377895832061768, "sampling/sampling_logp_difference/max": 0.6202878952026367, "sampling/sampling_logp_difference/mean": 0.0057983691804111, "step": 216 }, { "clip_ratio/high_max": 0.0001974168214170883, "clip_ratio/high_mean": 0.0001974168214170883, "clip_ratio/low_mean": 6.495545838131673e-05, "clip_ratio/low_min": 6.495545838131673e-05, "clip_ratio/region_mean": 0.000262372279798405, "completions/clipped_ratio": 0.0, "completions/max_length": 77.0, "completions/max_terminated_length": 77.0, "completions/mean_length": 52.99609375, "completions/mean_terminated_length": 52.99609375, "completions/min_length": 36.0, "completions/min_terminated_length": 36.0, "entropy": 0.11492264664007558, "epoch": 0.05814366998577525, "frac_reward_zero_std": 0.71875, "grad_norm": 0.4676933288574219, "kl": 0.21251611121826702, "learning_rate": 2.769680016269385e-07, "loss": -0.0036, "num_tokens": 14596754.0, "reward": 1.1171875, "reward_std": 0.11875930428504944, "rewards/equation_reward_func/mean": 0.1171875, "rewards/equation_reward_func/std": 0.32195815443992615, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.5096341371536255, "sampling/importance_sampling_ratio/mean": 0.9998860955238342, "sampling/importance_sampling_ratio/min": 0.5001150369644165, "sampling/sampling_logp_difference/max": 0.6929171085357666, "sampling/sampling_logp_difference/mean": 0.005857203155755997, "step": 218 }, { "clip_ratio/high_max": 0.00013001335901208222, "clip_ratio/high_mean": 0.00013001335901208222, "clip_ratio/low_mean": 9.567012117865185e-05, "clip_ratio/low_min": 9.567012117865185e-05, "clip_ratio/region_mean": 0.00022568348019073406, "completions/clipped_ratio": 0.0, "completions/max_length": 73.0, "completions/max_terminated_length": 73.0, "completions/mean_length": 53.59765625, "completions/mean_terminated_length": 53.59765625, "completions/min_length": 34.0, "completions/min_terminated_length": 34.0, "entropy": 0.11644557221896118, "epoch": 0.05867709815078236, "frac_reward_zero_std": 0.703125, "grad_norm": 0.5575525164604187, "kl": 0.19831780116591188, "learning_rate": 2.7338360423820327e-07, "loss": 0.0063, "num_tokens": 14702532.0, "reward": 1.09375, "reward_std": 0.12684723734855652, "rewards/equation_reward_func/mean": 0.09765625, "rewards/equation_reward_func/std": 0.29713961482048035, "rewards/format_reward_func/mean": 0.99609375, "rewards/format_reward_func/std": 0.06243881583213806, "sampling/importance_sampling_ratio/max": 1.5924863815307617, "sampling/importance_sampling_ratio/mean": 0.9996614456176758, "sampling/importance_sampling_ratio/min": 0.47730016708374023, "sampling/sampling_logp_difference/max": 0.7396097183227539, "sampling/sampling_logp_difference/mean": 0.005882314871996641, "step": 220 }, { "clip_ratio/high_max": 0.0003587727583686097, "clip_ratio/high_mean": 0.0003587727583686097, "clip_ratio/low_mean": 0.00016187850592864884, "clip_ratio/low_min": 0.00016187850592864884, "clip_ratio/region_mean": 0.0005206512642972586, "completions/clipped_ratio": 0.0, "completions/max_length": 74.0, "completions/max_terminated_length": 74.0, "completions/mean_length": 53.50390625, "completions/mean_terminated_length": 53.50390625, "completions/min_length": 37.0, "completions/min_terminated_length": 37.0, "entropy": 0.11813913865221871, "epoch": 0.05921052631578947, "frac_reward_zero_std": 0.625, "grad_norm": 1.0590949058532715, "kl": 0.3677462774018447, "learning_rate": 2.6979435071510956e-07, "loss": -0.0064, "num_tokens": 14808158.0, "reward": 1.125, "reward_std": 0.15932825207710266, "rewards/equation_reward_func/mean": 0.125, "rewards/equation_reward_func/std": 0.3310423493385315, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.5182225704193115, "sampling/importance_sampling_ratio/mean": 0.9999364614486694, "sampling/importance_sampling_ratio/min": 0.5411150455474854, "sampling/sampling_logp_difference/max": 0.6141233444213867, "sampling/sampling_logp_difference/mean": 0.006206444930285215, "step": 222 }, { "clip_ratio/high_max": 0.0003639115958422836, "clip_ratio/high_mean": 0.0003639115958422836, "clip_ratio/low_mean": 0.00020010410096599825, "clip_ratio/low_min": 0.00020010410096599825, "clip_ratio/region_mean": 0.0005640156968082818, "completions/clipped_ratio": 0.0, "completions/max_length": 82.0, "completions/max_terminated_length": 82.0, "completions/mean_length": 52.18359375, "completions/mean_terminated_length": 52.18359375, "completions/min_length": 33.0, "completions/min_terminated_length": 33.0, "entropy": 0.11953117636342843, "epoch": 0.059743954480796585, "frac_reward_zero_std": 0.59375, "grad_norm": 0.705763578414917, "kl": 0.19684387784865168, "learning_rate": 2.662009864473406e-07, "loss": 0.0075, "num_tokens": 14912820.0, "reward": 1.150390625, "reward_std": 0.18030321598052979, "rewards/equation_reward_func/mean": 0.150390625, "rewards/equation_reward_func/std": 0.35780346393585205, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.6097370386123657, "sampling/importance_sampling_ratio/mean": 1.0001099109649658, "sampling/importance_sampling_ratio/min": 0.4614678919315338, "sampling/sampling_logp_difference/max": 0.7733428478240967, "sampling/sampling_logp_difference/mean": 0.006311375647783279, "step": 224 }, { "clip_ratio/high_max": 9.85095975920558e-05, "clip_ratio/high_mean": 9.85095975920558e-05, "clip_ratio/low_mean": 0.00022559486226075224, "clip_ratio/low_min": 0.00022559486226075224, "clip_ratio/region_mean": 0.00032410445985280804, "completions/clipped_ratio": 0.0, "completions/max_length": 75.0, "completions/max_terminated_length": 75.0, "completions/mean_length": 52.91015625, "completions/mean_terminated_length": 52.91015625, "completions/min_length": 34.0, "completions/min_terminated_length": 34.0, "entropy": 0.11986736642817657, "epoch": 0.0602773826458037, "frac_reward_zero_std": 0.6640625, "grad_norm": 0.6192528605461121, "kl": 0.24484028832779992, "learning_rate": 2.626042576782687e-07, "loss": 0.0061, "num_tokens": 15123128.0, "reward": 1.134765625, "reward_std": 0.14946477115154266, "rewards/equation_reward_func/mean": 0.134765625, "rewards/equation_reward_func/std": 0.3403467684984207, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.7585115432739258, "sampling/importance_sampling_ratio/mean": 1.0001845359802246, "sampling/importance_sampling_ratio/min": 0.5394621193408966, "sampling/sampling_logp_difference/max": 0.6830110549926758, "sampling/sampling_logp_difference/mean": 0.00615977100096643, "step": 226 }, { "clip_ratio/high_max": 0.0002973029719820867, "clip_ratio/high_mean": 0.0002973029719820867, "clip_ratio/low_mean": 6.856257742684748e-05, "clip_ratio/low_min": 6.856257742684748e-05, "clip_ratio/region_mean": 0.0003658655494089342, "completions/clipped_ratio": 0.0, "completions/max_length": 71.0, "completions/max_terminated_length": 71.0, "completions/mean_length": 53.609375, "completions/mean_terminated_length": 53.609375, "completions/min_length": 31.0, "completions/min_terminated_length": 31.0, "entropy": 0.11842664269109567, "epoch": 0.060810810810810814, "frac_reward_zero_std": 0.734375, "grad_norm": 0.708135187625885, "kl": 0.1876696693814463, "learning_rate": 2.590049113499809e-07, "loss": -0.0091, "num_tokens": 15228832.0, "reward": 1.091796875, "reward_std": 0.11895088851451874, "rewards/equation_reward_func/mean": 0.091796875, "rewards/equation_reward_func/std": 0.289021372795105, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.6424556970596313, "sampling/importance_sampling_ratio/mean": 0.9999076128005981, "sampling/importance_sampling_ratio/min": 0.47103625535964966, "sampling/sampling_logp_difference/max": 0.7528202533721924, "sampling/sampling_logp_difference/mean": 0.006564016919583082, "step": 228 }, { "clip_ratio/high_max": 0.000291443365212116, "clip_ratio/high_mean": 0.000291443365212116, "clip_ratio/low_mean": 6.583160332714517e-05, "clip_ratio/low_min": 6.583160332714517e-05, "clip_ratio/region_mean": 0.00035727496853926114, "completions/clipped_ratio": 0.0, "completions/max_length": 69.0, "completions/max_terminated_length": 69.0, "completions/mean_length": 52.140625, "completions/mean_terminated_length": 52.140625, "completions/min_length": 36.0, "completions/min_terminated_length": 36.0, "entropy": 0.12662849927114117, "epoch": 0.06134423897581792, "frac_reward_zero_std": 0.6875, "grad_norm": 0.47451645135879517, "kl": 0.2256828004287349, "learning_rate": 2.5540369494815966e-07, "loss": -0.0013, "num_tokens": 15333496.0, "reward": 1.126953125, "reward_std": 0.1261277049779892, "rewards/equation_reward_func/mean": 0.12890625, "rewards/equation_reward_func/std": 0.33542385697364807, "rewards/format_reward_func/mean": 0.998046875, "rewards/format_reward_func/std": 0.04419417306780815, "sampling/importance_sampling_ratio/max": 1.6544182300567627, "sampling/importance_sampling_ratio/mean": 1.0002301931381226, "sampling/importance_sampling_ratio/min": 0.5614175796508789, "sampling/sampling_logp_difference/max": 0.5772902965545654, "sampling/sampling_logp_difference/mean": 0.00661100260913372, "step": 230 }, { "clip_ratio/high_max": 0.00010160589590668678, "clip_ratio/high_mean": 0.00010160589590668678, "clip_ratio/low_mean": 6.522905379016366e-05, "clip_ratio/low_min": 6.522905379016366e-05, "clip_ratio/region_mean": 0.00016683494969685044, "completions/clipped_ratio": 0.0, "completions/max_length": 71.0, "completions/max_terminated_length": 71.0, "completions/mean_length": 52.98828125, "completions/mean_terminated_length": 52.98828125, "completions/min_length": 34.0, "completions/min_terminated_length": 34.0, "entropy": 0.12193829255799453, "epoch": 0.061877667140825036, "frac_reward_zero_std": 0.734375, "grad_norm": 0.5908534526824951, "kl": 0.22970337400005925, "learning_rate": 2.5180135634685064e-07, "loss": 0.0012, "num_tokens": 15438778.0, "reward": 1.1171875, "reward_std": 0.11980462074279785, "rewards/equation_reward_func/mean": 0.1171875, "rewards/equation_reward_func/std": 0.32195815443992615, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.5204116106033325, "sampling/importance_sampling_ratio/mean": 0.9997380971908569, "sampling/importance_sampling_ratio/min": 0.49725833535194397, "sampling/sampling_logp_difference/max": 0.6986455917358398, "sampling/sampling_logp_difference/mean": 0.006525573320686817, "step": 232 }, { "clip_ratio/high_max": 0.00016493863788329891, "clip_ratio/high_mean": 0.00016493863788329891, "clip_ratio/low_mean": 0.0001652783765974972, "clip_ratio/low_min": 0.0001652783765974972, "clip_ratio/region_mean": 0.0003302170177145551, "completions/clipped_ratio": 0.0, "completions/max_length": 74.0, "completions/max_terminated_length": 74.0, "completions/mean_length": 52.349609375, "completions/mean_terminated_length": 52.349609375, "completions/min_length": 34.0, "completions/min_terminated_length": 34.0, "entropy": 0.1254527090738217, "epoch": 0.06241109530583215, "frac_reward_zero_std": 0.6875, "grad_norm": 0.6845598816871643, "kl": 0.1800268271731006, "learning_rate": 2.4819864365314934e-07, "loss": 0.0079, "num_tokens": 15543501.0, "reward": 1.16796875, "reward_std": 0.13776159286499023, "rewards/equation_reward_func/mean": 0.16796875, "rewards/equation_reward_func/std": 0.374204158782959, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.5828297138214111, "sampling/importance_sampling_ratio/mean": 1.000268578529358, "sampling/importance_sampling_ratio/min": 0.5083870887756348, "sampling/sampling_logp_difference/max": 0.6765121221542358, "sampling/sampling_logp_difference/mean": 0.00637659290805459, "step": 234 }, { "clip_ratio/high_max": 0.00019972173807521662, "clip_ratio/high_mean": 0.00019972173807521662, "clip_ratio/low_mean": 6.456824808588458e-05, "clip_ratio/low_min": 6.456824808588458e-05, "clip_ratio/region_mean": 0.0002642899861611012, "completions/clipped_ratio": 0.0, "completions/max_length": 71.0, "completions/max_terminated_length": 71.0, "completions/mean_length": 53.46484375, "completions/mean_terminated_length": 53.46484375, "completions/min_length": 29.0, "completions/min_terminated_length": 29.0, "entropy": 0.12084446619782183, "epoch": 0.06294452347083926, "frac_reward_zero_std": 0.640625, "grad_norm": 0.6229382753372192, "kl": 0.32917360506123966, "learning_rate": 2.445963050518403e-07, "loss": -0.0, "num_tokens": 15649187.0, "reward": 1.13671875, "reward_std": 0.1554545760154724, "rewards/equation_reward_func/mean": 0.138671875, "rewards/equation_reward_func/std": 0.34594178199768066, "rewards/format_reward_func/mean": 0.998046875, "rewards/format_reward_func/std": 0.04419417306780815, "sampling/importance_sampling_ratio/max": 1.5042505264282227, "sampling/importance_sampling_ratio/mean": 0.999774694442749, "sampling/importance_sampling_ratio/min": 0.4756012558937073, "sampling/sampling_logp_difference/max": 0.7431755065917969, "sampling/sampling_logp_difference/mean": 0.006509827449917793, "step": 236 }, { "clip_ratio/high_max": 3.2075955661841566e-05, "clip_ratio/high_mean": 3.2075955661841566e-05, "clip_ratio/low_mean": 3.1855248380452394e-05, "clip_ratio/low_min": 3.1855248380452394e-05, "clip_ratio/region_mean": 6.393120404229396e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 75.0, "completions/max_terminated_length": 75.0, "completions/mean_length": 52.603515625, "completions/mean_terminated_length": 52.603515625, "completions/min_length": 36.0, "completions/min_terminated_length": 36.0, "entropy": 0.12392547105749448, "epoch": 0.06347795163584637, "frac_reward_zero_std": 0.734375, "grad_norm": 0.5728958249092102, "kl": 0.22863260573811, "learning_rate": 2.4099508865001914e-07, "loss": -0.004, "num_tokens": 15754232.0, "reward": 1.15234375, "reward_std": 0.12145397812128067, "rewards/equation_reward_func/mean": 0.15234375, "rewards/equation_reward_func/std": 0.35970520973205566, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4579753875732422, "sampling/importance_sampling_ratio/mean": 1.0001283884048462, "sampling/importance_sampling_ratio/min": 0.47823217511177063, "sampling/sampling_logp_difference/max": 0.7376589775085449, "sampling/sampling_logp_difference/mean": 0.006175911519676447, "step": 238 }, { "clip_ratio/high_max": 9.573096758686006e-05, "clip_ratio/high_mean": 9.573096758686006e-05, "clip_ratio/low_mean": 9.874854650762345e-05, "clip_ratio/low_min": 9.874854650762345e-05, "clip_ratio/region_mean": 0.00019447951409448352, "completions/clipped_ratio": 0.0, "completions/max_length": 68.0, "completions/max_terminated_length": 68.0, "completions/mean_length": 53.09375, "completions/mean_terminated_length": 53.09375, "completions/min_length": 36.0, "completions/min_terminated_length": 36.0, "entropy": 0.117175265111857, "epoch": 0.06401137980085349, "frac_reward_zero_std": 0.671875, "grad_norm": 0.5250370502471924, "kl": 0.28859515984853107, "learning_rate": 2.3739574232173134e-07, "loss": 0.0005, "num_tokens": 15859560.0, "reward": 1.162109375, "reward_std": 0.14729177951812744, "rewards/equation_reward_func/mean": 0.162109375, "rewards/equation_reward_func/std": 0.3689115643501282, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.5415260791778564, "sampling/importance_sampling_ratio/mean": 1.0000598430633545, "sampling/importance_sampling_ratio/min": 0.5174270868301392, "sampling/sampling_logp_difference/max": 0.6588866710662842, "sampling/sampling_logp_difference/mean": 0.006172684486955404, "step": 240 }, { "clip_ratio/high_max": 3.264133637357089e-05, "clip_ratio/high_mean": 3.264133637357089e-05, "clip_ratio/low_mean": 9.471062609615426e-05, "clip_ratio/low_min": 9.471062609615426e-05, "clip_ratio/region_mean": 0.00012735196246972514, "completions/clipped_ratio": 0.0, "completions/max_length": 71.5, "completions/max_terminated_length": 71.5, "completions/mean_length": 53.861328125, "completions/mean_terminated_length": 53.861328125, "completions/min_length": 36.5, "completions/min_terminated_length": 36.5, "entropy": 0.12318918564253384, "epoch": 0.0645448079658606, "frac_reward_zero_std": 0.734375, "grad_norm": 0.4942455589771271, "kl": 0.23004327176345718, "learning_rate": 2.3379901355265936e-07, "loss": 0.0022, "num_tokens": 16071202.0, "reward": 1.1396484375, "reward_std": 0.11553288251161575, "rewards/equation_reward_func/mean": 0.140625, "rewards/equation_reward_func/std": 0.34502755105495453, "rewards/format_reward_func/mean": 0.9990234375, "rewards/format_reward_func/std": 0.022097086533904076, "sampling/importance_sampling_ratio/max": 1.4460033774375916, "sampling/importance_sampling_ratio/mean": 1.0000775754451752, "sampling/importance_sampling_ratio/min": 0.5052160322666168, "sampling/sampling_logp_difference/max": 0.7033942937850952, "sampling/sampling_logp_difference/mean": 0.006043520988896489, "step": 242 }, { "clip_ratio/high_max": 6.313164097567399e-05, "clip_ratio/high_mean": 6.313164097567399e-05, "clip_ratio/low_mean": 0.00012851631294728981, "clip_ratio/low_min": 0.00012851631294728981, "clip_ratio/region_mean": 0.00019164795392296382, "completions/clipped_ratio": 0.0, "completions/max_length": 81.0, "completions/max_terminated_length": 81.0, "completions/mean_length": 53.490234375, "completions/mean_terminated_length": 53.490234375, "completions/min_length": 34.0, "completions/min_terminated_length": 34.0, "entropy": 0.12224643304944038, "epoch": 0.0650782361308677, "frac_reward_zero_std": 0.71875, "grad_norm": 0.5327535271644592, "kl": 0.1954546593543556, "learning_rate": 2.3020564928489041e-07, "loss": -0.0005, "num_tokens": 16176717.0, "reward": 1.09375, "reward_std": 0.12671303749084473, "rewards/equation_reward_func/mean": 0.09375, "rewards/equation_reward_func/std": 0.29176566004753113, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.5413131713867188, "sampling/importance_sampling_ratio/mean": 1.0001628398895264, "sampling/importance_sampling_ratio/min": 0.6364158391952515, "sampling/sampling_logp_difference/max": 0.4519031047821045, "sampling/sampling_logp_difference/mean": 0.006325087510049343, "step": 244 }, { "clip_ratio/high_max": 6.61559428812729e-05, "clip_ratio/high_mean": 6.61559428812729e-05, "clip_ratio/low_mean": 0.00016481440333235596, "clip_ratio/low_min": 0.00016481440333235596, "clip_ratio/region_mean": 0.00023097034621362886, "completions/clipped_ratio": 0.0, "completions/max_length": 74.0, "completions/max_terminated_length": 74.0, "completions/mean_length": 52.734375, "completions/mean_terminated_length": 52.734375, "completions/min_length": 36.0, "completions/min_terminated_length": 36.0, "entropy": 0.12224224023520947, "epoch": 0.06561166429587482, "frac_reward_zero_std": 0.734375, "grad_norm": 0.5621053576469421, "kl": 0.2635859563532803, "learning_rate": 2.2661639576179676e-07, "loss": -0.0083, "num_tokens": 16281669.0, "reward": 1.173828125, "reward_std": 0.11270461976528168, "rewards/equation_reward_func/mean": 0.17578125, "rewards/equation_reward_func/std": 0.3810062110424042, "rewards/format_reward_func/mean": 0.998046875, "rewards/format_reward_func/std": 0.04419417306780815, "sampling/importance_sampling_ratio/max": 1.4159756898880005, "sampling/importance_sampling_ratio/mean": 1.0000406503677368, "sampling/importance_sampling_ratio/min": 0.5447098612785339, "sampling/sampling_logp_difference/max": 0.6075019836425781, "sampling/sampling_logp_difference/mean": 0.005927799269556999, "step": 246 }, { "clip_ratio/high_max": 0.00013626467100241117, "clip_ratio/high_mean": 0.00013626467100241117, "clip_ratio/low_mean": 0.0002326449547480378, "clip_ratio/low_min": 0.0002326449547480378, "clip_ratio/region_mean": 0.00036890962575044896, "completions/clipped_ratio": 0.0, "completions/max_length": 78.0, "completions/max_terminated_length": 78.0, "completions/mean_length": 52.96484375, "completions/mean_terminated_length": 52.96484375, "completions/min_length": 30.0, "completions/min_terminated_length": 30.0, "entropy": 0.12200754197935264, "epoch": 0.06614509246088193, "frac_reward_zero_std": 0.640625, "grad_norm": 1.1268446445465088, "kl": 0.28940939489338136, "learning_rate": 2.2303199837306153e-07, "loss": 0.0127, "num_tokens": 16386787.0, "reward": 1.138671875, "reward_std": 0.1637316346168518, "rewards/equation_reward_func/mean": 0.140625, "rewards/equation_reward_func/std": 0.3479743003845215, "rewards/format_reward_func/mean": 0.998046875, "rewards/format_reward_func/std": 0.04419417306780815, "sampling/importance_sampling_ratio/max": 1.6521880626678467, "sampling/importance_sampling_ratio/mean": 1.000035047531128, "sampling/importance_sampling_ratio/min": 0.5477961897850037, "sampling/sampling_logp_difference/max": 0.6018519401550293, "sampling/sampling_logp_difference/mean": 0.006177326198667288, "step": 248 }, { "clip_ratio/high_max": 6.517285105979277e-05, "clip_ratio/high_mean": 6.517285105979277e-05, "clip_ratio/low_mean": 0.00036028153004331724, "clip_ratio/low_min": 0.00036028153004331724, "clip_ratio/region_mean": 0.00042545438110311, "completions/clipped_ratio": 0.0, "completions/max_length": 70.0, "completions/max_terminated_length": 70.0, "completions/mean_length": 52.390625, "completions/mean_terminated_length": 52.390625, "completions/min_length": 33.0, "completions/min_terminated_length": 33.0, "entropy": 0.1172566707763407, "epoch": 0.06667852062588904, "frac_reward_zero_std": 0.6875, "grad_norm": 0.5600212216377258, "kl": 0.22640964802768496, "learning_rate": 2.194532014998817e-07, "loss": -0.0154, "num_tokens": 16491555.0, "reward": 1.0859375, "reward_std": 0.1346687376499176, "rewards/equation_reward_func/mean": 0.087890625, "rewards/equation_reward_func/std": 0.2834126651287079, "rewards/format_reward_func/mean": 0.998046875, "rewards/format_reward_func/std": 0.04419417306780815, "sampling/importance_sampling_ratio/max": 1.541048288345337, "sampling/importance_sampling_ratio/mean": 0.9998010396957397, "sampling/importance_sampling_ratio/min": 0.4184038043022156, "sampling/sampling_logp_difference/max": 0.8713083267211914, "sampling/sampling_logp_difference/mean": 0.005250845570117235, "step": 250 }, { "clip_ratio/high_max": 0.00013368520174278982, "clip_ratio/high_mean": 0.00013368520174278982, "clip_ratio/low_mean": 0.00043396216480889253, "clip_ratio/low_min": 0.00043396216480889253, "clip_ratio/region_mean": 0.0005676473665516824, "completions/clipped_ratio": 0.0, "completions/max_length": 69.0, "completions/max_terminated_length": 69.0, "completions/mean_length": 53.234375, "completions/mean_terminated_length": 53.234375, "completions/min_length": 36.0, "completions/min_terminated_length": 36.0, "entropy": 0.11685328806440036, "epoch": 0.06721194879089616, "frac_reward_zero_std": 0.625, "grad_norm": 0.6178267002105713, "kl": 0.48353597418301636, "learning_rate": 2.1588074836038071e-07, "loss": 0.0135, "num_tokens": 16596835.0, "reward": 1.1484375, "reward_std": 0.16728200018405914, "rewards/equation_reward_func/mean": 0.150390625, "rewards/equation_reward_func/std": 0.35780346393585205, "rewards/format_reward_func/mean": 0.998046875, "rewards/format_reward_func/std": 0.04419417306780815, "sampling/importance_sampling_ratio/max": 1.5711119174957275, "sampling/importance_sampling_ratio/mean": 0.9999057650566101, "sampling/importance_sampling_ratio/min": 0.6018173098564148, "sampling/sampling_logp_difference/max": 0.5078012943267822, "sampling/sampling_logp_difference/mean": 0.005669458769261837, "step": 252 }, { "clip_ratio/high_max": 0.00013041867189005844, "clip_ratio/high_mean": 0.00013041867189005844, "clip_ratio/low_mean": 0.0001985825365409255, "clip_ratio/low_min": 0.0001985825365409255, "clip_ratio/region_mean": 0.00032900120843098394, "completions/clipped_ratio": 0.0, "completions/max_length": 81.0, "completions/max_terminated_length": 81.0, "completions/mean_length": 53.201171875, "completions/mean_terminated_length": 53.201171875, "completions/min_length": 36.0, "completions/min_terminated_length": 36.0, "entropy": 0.11372616328299046, "epoch": 0.06774537695590327, "frac_reward_zero_std": 0.671875, "grad_norm": 0.6195980310440063, "kl": 0.20957040414214134, "learning_rate": 2.1231538085526204e-07, "loss": -0.0052, "num_tokens": 16702090.0, "reward": 1.173828125, "reward_std": 0.14814674854278564, "rewards/equation_reward_func/mean": 0.173828125, "rewards/equation_reward_func/std": 0.3793322443962097, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4368383884429932, "sampling/importance_sampling_ratio/mean": 1.0000479221343994, "sampling/importance_sampling_ratio/min": 0.5573049187660217, "sampling/sampling_logp_difference/max": 0.5846426486968994, "sampling/sampling_logp_difference/mean": 0.005298153031617403, "step": 254 }, { "clip_ratio/high_max": 0.00016930111774450375, "clip_ratio/high_mean": 0.00016930111774450375, "clip_ratio/low_mean": 0.0002204755631585916, "clip_ratio/low_min": 0.0002204755631585916, "clip_ratio/region_mean": 0.0003897766776693364, "completions/clipped_ratio": 0.0, "completions/max_length": 87.0, "completions/max_terminated_length": 87.0, "completions/mean_length": 54.55078125, "completions/mean_terminated_length": 54.55078125, "completions/min_length": 36.0, "completions/min_terminated_length": 36.0, "entropy": 0.12089426215324137, "epoch": 0.06827880512091039, "frac_reward_zero_std": 0.6875, "grad_norm": 0.7291125059127808, "kl": 0.2326575161682235, "learning_rate": 2.0875783941373686e-07, "loss": 0.0027, "num_tokens": 16808332.0, "reward": 1.115234375, "reward_std": 0.1349339485168457, "rewards/equation_reward_func/mean": 0.1171875, "rewards/equation_reward_func/std": 0.32195815443992615, "rewards/format_reward_func/mean": 0.998046875, "rewards/format_reward_func/std": 0.04419417306780815, "sampling/importance_sampling_ratio/max": 1.592735767364502, "sampling/importance_sampling_ratio/mean": 1.0001572370529175, "sampling/importance_sampling_ratio/min": 0.6112369894981384, "sampling/sampling_logp_difference/max": 0.49227046966552734, "sampling/sampling_logp_difference/mean": 0.006166752427816391, "step": 256 }, { "clip_ratio/high_max": 6.400825805030763e-05, "clip_ratio/high_mean": 6.400825805030763e-05, "clip_ratio/low_mean": 6.572409701119695e-05, "clip_ratio/low_min": 6.572409701119695e-05, "clip_ratio/region_mean": 0.00012973235506150458, "completions/clipped_ratio": 0.0, "completions/max_length": 77.0, "completions/max_terminated_length": 77.0, "completions/mean_length": 53.673828125, "completions/mean_terminated_length": 53.673828125, "completions/min_length": 35.5, "completions/min_terminated_length": 35.5, "entropy": 0.11469285790291098, "epoch": 0.0688122332859175, "frac_reward_zero_std": 0.671875, "grad_norm": 0.6034818291664124, "kl": 0.21051878440711233, "learning_rate": 2.052088628397572e-07, "loss": 0.0046, "num_tokens": 17019478.0, "reward": 1.140625, "reward_std": 0.14814674109220505, "rewards/equation_reward_func/mean": 0.140625, "rewards/equation_reward_func/std": 0.3478719890117645, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.6311516761779785, "sampling/importance_sampling_ratio/mean": 1.0000055432319641, "sampling/importance_sampling_ratio/min": 0.5794435143470764, "sampling/sampling_logp_difference/max": 0.5485678911209106, "sampling/sampling_logp_difference/mean": 0.005461497465148568, "step": 258 }, { "clip_ratio/high_max": 0.00019554619858455326, "clip_ratio/high_mean": 0.00019554619858455326, "clip_ratio/low_mean": 9.79065597574744e-05, "clip_ratio/low_min": 9.79065597574744e-05, "clip_ratio/region_mean": 0.00029345275834202766, "completions/clipped_ratio": 0.0, "completions/max_length": 71.0, "completions/max_terminated_length": 71.0, "completions/mean_length": 53.521484375, "completions/mean_terminated_length": 53.521484375, "completions/min_length": 36.0, "completions/min_terminated_length": 36.0, "entropy": 0.11641324021750027, "epoch": 0.0693456614509246, "frac_reward_zero_std": 0.671875, "grad_norm": 0.5642570853233337, "kl": 0.2255418917371167, "learning_rate": 2.0166918815858688e-07, "loss": -0.0029, "num_tokens": 17124913.0, "reward": 1.16796875, "reward_std": 0.14190170168876648, "rewards/equation_reward_func/mean": 0.16796875, "rewards/equation_reward_func/std": 0.374204158782959, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.7392843961715698, "sampling/importance_sampling_ratio/mean": 0.9999298453330994, "sampling/importance_sampling_ratio/min": 0.4831935167312622, "sampling/sampling_logp_difference/max": 0.7273380756378174, "sampling/sampling_logp_difference/mean": 0.005574048962444067, "step": 260 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0002266580381223725, "clip_ratio/low_min": 0.0002266580381223725, "clip_ratio/region_mean": 0.0002266580381223725, "completions/clipped_ratio": 0.0, "completions/max_length": 70.0, "completions/max_terminated_length": 70.0, "completions/mean_length": 54.185546875, "completions/mean_terminated_length": 54.185546875, "completions/min_length": 36.0, "completions/min_terminated_length": 36.0, "entropy": 0.11455756094720629, "epoch": 0.06987908961593173, "frac_reward_zero_std": 0.65625, "grad_norm": 0.550462543964386, "kl": 0.17767776879999372, "learning_rate": 1.9813955046374102e-07, "loss": -0.0048, "num_tokens": 17230824.0, "reward": 1.1953125, "reward_std": 0.1442737877368927, "rewards/equation_reward_func/mean": 0.1953125, "rewards/equation_reward_func/std": 0.3968288004398346, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.8483010530471802, "sampling/importance_sampling_ratio/mean": 0.999902606010437, "sampling/importance_sampling_ratio/min": 0.5373868346214294, "sampling/sampling_logp_difference/max": 0.6210370063781738, "sampling/sampling_logp_difference/mean": 0.005659033544361591, "step": 262 }, { "clip_ratio/high_max": 6.5365269417978e-05, "clip_ratio/high_mean": 6.5365269417978e-05, "clip_ratio/low_mean": 9.701474078206552e-05, "clip_ratio/low_min": 9.701474078206552e-05, "clip_ratio/region_mean": 0.00016238001020004353, "completions/clipped_ratio": 0.0, "completions/max_length": 77.0, "completions/max_terminated_length": 77.0, "completions/mean_length": 53.888671875, "completions/mean_terminated_length": 53.888671875, "completions/min_length": 36.0, "completions/min_terminated_length": 36.0, "entropy": 0.11500472492641872, "epoch": 0.07041251778093884, "frac_reward_zero_std": 0.6875, "grad_norm": 0.7422778010368347, "kl": 0.2729740840279394, "learning_rate": 1.946206827643275e-07, "loss": 0.0041, "num_tokens": 17336551.0, "reward": 1.181640625, "reward_std": 0.1434762179851532, "rewards/equation_reward_func/mean": 0.181640625, "rewards/equation_reward_func/std": 0.38592514395713806, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.5066125392913818, "sampling/importance_sampling_ratio/mean": 0.9999542832374573, "sampling/importance_sampling_ratio/min": 0.42032745480537415, "sampling/sampling_logp_difference/max": 0.8667211532592773, "sampling/sampling_logp_difference/mean": 0.005726358853280544, "step": 264 }, { "clip_ratio/high_max": 0.00012987014876368144, "clip_ratio/high_mean": 0.00012987014876368144, "clip_ratio/low_mean": 0.00012680440947103003, "clip_ratio/low_min": 0.00012680440947103003, "clip_ratio/region_mean": 0.00025667455823471147, "completions/clipped_ratio": 0.0, "completions/max_length": 72.0, "completions/max_terminated_length": 72.0, "completions/mean_length": 54.5390625, "completions/mean_terminated_length": 54.5390625, "completions/min_length": 25.0, "completions/min_terminated_length": 25.0, "entropy": 0.11781905467311542, "epoch": 0.07094594594594594, "frac_reward_zero_std": 0.703125, "grad_norm": 0.5870804190635681, "kl": 0.18947985106044346, "learning_rate": 1.9111331583282103e-07, "loss": 0.0051, "num_tokens": 17442779.0, "reward": 1.12890625, "reward_std": 0.13250252604484558, "rewards/equation_reward_func/mean": 0.12890625, "rewards/equation_reward_func/std": 0.33542385697364807, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.785035490989685, "sampling/importance_sampling_ratio/mean": 0.9999924302101135, "sampling/importance_sampling_ratio/min": 0.47538992762565613, "sampling/sampling_logp_difference/max": 0.7436199188232422, "sampling/sampling_logp_difference/mean": 0.005903167650103569, "step": 266 }, { "clip_ratio/high_max": 0.00012978605809621513, "clip_ratio/high_mean": 0.00012978605809621513, "clip_ratio/low_mean": 0.00015856400972956585, "clip_ratio/low_min": 0.00015856400972956585, "clip_ratio/region_mean": 0.00028835006782578095, "completions/clipped_ratio": 0.0, "completions/max_length": 74.0, "completions/max_terminated_length": 74.0, "completions/mean_length": 54.20703125, "completions/mean_terminated_length": 54.20703125, "completions/min_length": 35.0, "completions/min_terminated_length": 35.0, "entropy": 0.11669882759451866, "epoch": 0.07147937411095306, "frac_reward_zero_std": 0.71875, "grad_norm": 0.7162891626358032, "kl": 0.2612205685840713, "learning_rate": 1.8761817805330195e-07, "loss": -0.0032, "num_tokens": 17548757.0, "reward": 1.115234375, "reward_std": 0.12164750695228577, "rewards/equation_reward_func/mean": 0.115234375, "rewards/equation_reward_func/std": 0.3196168541908264, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.7249046564102173, "sampling/importance_sampling_ratio/mean": 0.9998591542243958, "sampling/importance_sampling_ratio/min": 0.4793323278427124, "sampling/sampling_logp_difference/max": 0.7353610992431641, "sampling/sampling_logp_difference/mean": 0.006094635929912329, "step": 268 }, { "clip_ratio/high_max": 0.0005107695793008639, "clip_ratio/high_mean": 0.0005107695793008639, "clip_ratio/low_mean": 0.00022279931646254327, "clip_ratio/low_min": 0.00022279931646254327, "clip_ratio/region_mean": 0.000733568902230925, "completions/clipped_ratio": 0.0, "completions/max_length": 75.0, "completions/max_terminated_length": 75.0, "completions/mean_length": 54.283203125, "completions/mean_terminated_length": 54.283203125, "completions/min_length": 36.0, "completions/min_terminated_length": 36.0, "entropy": 0.11902402403453986, "epoch": 0.07201280227596017, "frac_reward_zero_std": 0.734375, "grad_norm": 0.37476852536201477, "kl": 0.22781566406289736, "learning_rate": 1.8413599527019018e-07, "loss": 0.001, "num_tokens": 17654854.0, "reward": 1.146484375, "reward_std": 0.12348721921443939, "rewards/equation_reward_func/mean": 0.146484375, "rewards/equation_reward_func/std": 0.35393697023391724, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.42414128780365, "sampling/importance_sampling_ratio/mean": 0.9999381899833679, "sampling/importance_sampling_ratio/min": 0.3701840937137604, "sampling/sampling_logp_difference/max": 0.9937548637390137, "sampling/sampling_logp_difference/mean": 0.0060479482635855675, "step": 270 }, { "clip_ratio/high_max": 3.0358228185731503e-05, "clip_ratio/high_mean": 3.0358228185731503e-05, "clip_ratio/low_mean": 6.29265883213116e-05, "clip_ratio/low_min": 6.29265883213116e-05, "clip_ratio/region_mean": 9.328481650704311e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 76.0, "completions/max_terminated_length": 76.0, "completions/mean_length": 55.328125, "completions/mean_terminated_length": 55.328125, "completions/min_length": 38.0, "completions/min_terminated_length": 38.0, "entropy": 0.1182148756666316, "epoch": 0.07254623044096728, "frac_reward_zero_std": 0.640625, "grad_norm": 0.7100948095321655, "kl": 0.2611344123466147, "learning_rate": 1.806674906375079e-07, "loss": -0.0018, "num_tokens": 17761686.0, "reward": 1.140625, "reward_std": 0.15012383460998535, "rewards/equation_reward_func/mean": 0.142578125, "rewards/equation_reward_func/std": 0.3499840497970581, "rewards/format_reward_func/mean": 0.998046875, "rewards/format_reward_func/std": 0.04419417306780815, "sampling/importance_sampling_ratio/max": 1.5610944032669067, "sampling/importance_sampling_ratio/mean": 0.9999486804008484, "sampling/importance_sampling_ratio/min": 0.5355713367462158, "sampling/sampling_logp_difference/max": 0.6244211196899414, "sampling/sampling_logp_difference/mean": 0.005923059768974781, "step": 272 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.709608068275783e-05, "clip_ratio/low_min": 6.709608068275783e-05, "clip_ratio/region_mean": 6.709608068275783e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 75.5, "completions/max_terminated_length": 75.5, "completions/mean_length": 54.0654296875, "completions/mean_terminated_length": 54.0654296875, "completions/min_length": 34.0, "completions/min_terminated_length": 34.0, "entropy": 0.1252065480997165, "epoch": 0.0730796586059744, "frac_reward_zero_std": 0.6328125, "grad_norm": 0.8991405367851257, "kl": 0.24130096824632752, "learning_rate": 1.7721338446869976e-07, "loss": 0.0033, "num_tokens": 17973305.0, "reward": 1.189453125, "reward_std": 0.15672777593135834, "rewards/equation_reward_func/mean": 0.1904296875, "rewards/equation_reward_func/std": 0.3925798833370209, "rewards/format_reward_func/mean": 0.9990234375, "rewards/format_reward_func/std": 0.022097086533904076, "sampling/importance_sampling_ratio/max": 1.758608877658844, "sampling/importance_sampling_ratio/mean": 1.0000474452972412, "sampling/importance_sampling_ratio/min": 0.5109315365552902, "sampling/sampling_logp_difference/max": 0.7791448831558228, "sampling/sampling_logp_difference/mean": 0.006075352896004915, "step": 274 }, { "clip_ratio/high_max": 0.00025671912590041757, "clip_ratio/high_mean": 0.00025671912590041757, "clip_ratio/low_mean": 0.0004464233708050516, "clip_ratio/low_min": 0.0004464233708050516, "clip_ratio/region_mean": 0.0007031424967054692, "completions/clipped_ratio": 0.0, "completions/max_length": 74.0, "completions/max_terminated_length": 74.0, "completions/mean_length": 53.328125, "completions/mean_terminated_length": 53.328125, "completions/min_length": 33.0, "completions/min_terminated_length": 33.0, "entropy": 0.11954044260912472, "epoch": 0.07361308677098151, "frac_reward_zero_std": 0.703125, "grad_norm": 0.5922167301177979, "kl": 0.25013600910703343, "learning_rate": 1.7377439408704392e-07, "loss": -0.0024, "num_tokens": 18078657.0, "reward": 1.17578125, "reward_std": 0.11974846571683884, "rewards/equation_reward_func/mean": 0.17578125, "rewards/equation_reward_func/std": 0.3810062110424042, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.5618880987167358, "sampling/importance_sampling_ratio/mean": 0.9997606873512268, "sampling/importance_sampling_ratio/min": 0.4763152599334717, "sampling/sampling_logp_difference/max": 0.7416753768920898, "sampling/sampling_logp_difference/mean": 0.005760817788541317, "step": 276 }, { "clip_ratio/high_max": 6.727786967530847e-05, "clip_ratio/high_mean": 6.727786967530847e-05, "clip_ratio/low_mean": 9.992629768223398e-05, "clip_ratio/low_min": 9.992629768223398e-05, "clip_ratio/region_mean": 0.00016720416735754244, "completions/clipped_ratio": 0.0, "completions/max_length": 74.0, "completions/max_terminated_length": 74.0, "completions/mean_length": 54.8671875, "completions/mean_terminated_length": 54.8671875, "completions/min_length": 33.0, "completions/min_terminated_length": 33.0, "entropy": 0.11932587520115906, "epoch": 0.07414651493598862, "frac_reward_zero_std": 0.765625, "grad_norm": 0.6116552352905273, "kl": 0.19230408842364946, "learning_rate": 1.7035123367668323e-07, "loss": -0.0021, "num_tokens": 18185133.0, "reward": 1.126953125, "reward_std": 0.10245292633771896, "rewards/equation_reward_func/mean": 0.126953125, "rewards/equation_reward_func/std": 0.33324605226516724, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.7090688943862915, "sampling/importance_sampling_ratio/mean": 0.9999284148216248, "sampling/importance_sampling_ratio/min": 0.41860297322273254, "sampling/sampling_logp_difference/max": 0.8708324432373047, "sampling/sampling_logp_difference/mean": 0.006055726669728756, "step": 278 }, { "clip_ratio/high_max": 0.0001274182965668539, "clip_ratio/high_mean": 0.0001274182965668539, "clip_ratio/low_mean": 0.00015632986711959043, "clip_ratio/low_min": 0.00015632986711959043, "clip_ratio/region_mean": 0.00028374816368644434, "completions/clipped_ratio": 0.0, "completions/max_length": 68.0, "completions/max_terminated_length": 68.0, "completions/mean_length": 53.59765625, "completions/mean_terminated_length": 53.59765625, "completions/min_length": 35.0, "completions/min_terminated_length": 35.0, "entropy": 0.10979976960354382, "epoch": 0.07467994310099574, "frac_reward_zero_std": 0.71875, "grad_norm": 0.7363688349723816, "kl": 0.21388624215291607, "learning_rate": 1.6694461413430893e-07, "loss": 0.0044, "num_tokens": 18290639.0, "reward": 1.18359375, "reward_std": 0.12020279467105865, "rewards/equation_reward_func/mean": 0.18359375, "rewards/equation_reward_func/std": 0.3875311613082886, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.6355280876159668, "sampling/importance_sampling_ratio/mean": 1.000213623046875, "sampling/importance_sampling_ratio/min": 0.5373046398162842, "sampling/sampling_logp_difference/max": 0.621190071105957, "sampling/sampling_logp_difference/mean": 0.005433195736259222, "step": 280 }, { "clip_ratio/high_max": 0.00012727488905915784, "clip_ratio/high_mean": 0.00012727488905915784, "clip_ratio/low_mean": 0.00022515782297381925, "clip_ratio/low_min": 0.00022515782297381925, "clip_ratio/region_mean": 0.0003524327120329771, "completions/clipped_ratio": 0.0, "completions/max_length": 73.0, "completions/max_terminated_length": 73.0, "completions/mean_length": 54.451171875, "completions/mean_terminated_length": 54.451171875, "completions/min_length": 36.0, "completions/min_terminated_length": 36.0, "entropy": 0.11184248907698525, "epoch": 0.07521337126600285, "frac_reward_zero_std": 0.625, "grad_norm": 0.7730035185813904, "kl": 0.24625489198499256, "learning_rate": 1.6355524292152684e-07, "loss": -0.0023, "num_tokens": 18396798.0, "reward": 1.16796875, "reward_std": 0.16813695430755615, "rewards/equation_reward_func/mean": 0.169921875, "rewards/equation_reward_func/std": 0.3759314715862274, "rewards/format_reward_func/mean": 0.998046875, "rewards/format_reward_func/std": 0.04419417306780815, "sampling/importance_sampling_ratio/max": 1.709104061126709, "sampling/importance_sampling_ratio/mean": 1.0000386238098145, "sampling/importance_sampling_ratio/min": 0.48130878806114197, "sampling/sampling_logp_difference/max": 0.7312462329864502, "sampling/sampling_logp_difference/mean": 0.005882616620510817, "step": 282 }, { "clip_ratio/high_max": 0.00022572680609300733, "clip_ratio/high_mean": 0.00022572680609300733, "clip_ratio/low_mean": 0.0002221358267383443, "clip_ratio/low_min": 0.0002221358267383443, "clip_ratio/region_mean": 0.0004478626295975927, "completions/clipped_ratio": 0.0, "completions/max_length": 119.0, "completions/max_terminated_length": 119.0, "completions/mean_length": 53.83203125, "completions/mean_terminated_length": 53.83203125, "completions/min_length": 36.0, "completions/min_terminated_length": 36.0, "entropy": 0.11374843782848781, "epoch": 0.07574679943100995, "frac_reward_zero_std": 0.65625, "grad_norm": 0.7327376008033752, "kl": 0.2505280135406388, "learning_rate": 1.6018382391793722e-07, "loss": 0.0008, "num_tokens": 18502408.0, "reward": 1.1171875, "reward_std": 0.1536710113286972, "rewards/equation_reward_func/mean": 0.119140625, "rewards/equation_reward_func/std": 0.32427072525024414, "rewards/format_reward_func/mean": 0.998046875, "rewards/format_reward_func/std": 0.04419417306780815, "sampling/importance_sampling_ratio/max": 1.6212329864501953, "sampling/importance_sampling_ratio/mean": 0.9998156428337097, "sampling/importance_sampling_ratio/min": 0.47825267910957336, "sampling/sampling_logp_difference/max": 0.7376160621643066, "sampling/sampling_logp_difference/mean": 0.005457405000925064, "step": 284 }, { "clip_ratio/high_max": 0.00019255069976982972, "clip_ratio/high_mean": 0.00019255069976982972, "clip_ratio/low_mean": 0.0001325318201755484, "clip_ratio/low_min": 0.0001325318201755484, "clip_ratio/region_mean": 0.0003250825199453781, "completions/clipped_ratio": 0.0, "completions/max_length": 68.0, "completions/max_terminated_length": 68.0, "completions/mean_length": 53.380859375, "completions/mean_terminated_length": 53.380859375, "completions/min_length": 34.0, "completions/min_terminated_length": 34.0, "entropy": 0.11636582741306888, "epoch": 0.07628022759601708, "frac_reward_zero_std": 0.703125, "grad_norm": 0.6755468845367432, "kl": 0.3760374633388387, "learning_rate": 1.5683105727495778e-07, "loss": -0.0023, "num_tokens": 18607803.0, "reward": 1.12890625, "reward_std": 0.12402082234621048, "rewards/equation_reward_func/mean": 0.12890625, "rewards/equation_reward_func/std": 0.33542385697364807, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.724694013595581, "sampling/importance_sampling_ratio/mean": 1.0000168085098267, "sampling/importance_sampling_ratio/min": 0.6067004799842834, "sampling/sampling_logp_difference/max": 0.5450496673583984, "sampling/sampling_logp_difference/mean": 0.0060112448409199715, "step": 286 }, { "clip_ratio/high_max": 0.00016007068269472156, "clip_ratio/high_mean": 0.00016007068269472156, "clip_ratio/low_mean": 0.00012752274051308632, "clip_ratio/low_min": 0.00012752274051308632, "clip_ratio/region_mean": 0.0002875934232078079, "completions/clipped_ratio": 0.0, "completions/max_length": 82.0, "completions/max_terminated_length": 82.0, "completions/mean_length": 54.20703125, "completions/mean_terminated_length": 54.20703125, "completions/min_length": 31.0, "completions/min_terminated_length": 31.0, "entropy": 0.1160916495654318, "epoch": 0.07681365576102418, "frac_reward_zero_std": 0.671875, "grad_norm": 0.6371228694915771, "kl": 0.20908190992971262, "learning_rate": 1.5349763927042168e-07, "loss": 0.001, "num_tokens": 18713781.0, "reward": 1.140625, "reward_std": 0.14104920625686646, "rewards/equation_reward_func/mean": 0.140625, "rewards/equation_reward_func/std": 0.3479743003845215, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.633782982826233, "sampling/importance_sampling_ratio/mean": 1.000022530555725, "sampling/importance_sampling_ratio/min": 0.4739110767841339, "sampling/sampling_logp_difference/max": 0.7467355728149414, "sampling/sampling_logp_difference/mean": 0.006055853795260191, "step": 288 }, { "clip_ratio/high_max": 6.467687651618487e-05, "clip_ratio/high_mean": 6.467687651618487e-05, "clip_ratio/low_mean": 6.382815384616454e-05, "clip_ratio/low_min": 6.382815384616454e-05, "clip_ratio/region_mean": 0.0001285050303623494, "completions/clipped_ratio": 0.0, "completions/max_length": 82.5, "completions/max_terminated_length": 82.5, "completions/mean_length": 53.6474609375, "completions/mean_terminated_length": 53.6474609375, "completions/min_length": 33.5, "completions/min_terminated_length": 33.5, "entropy": 0.1184024533463849, "epoch": 0.07734708392603129, "frac_reward_zero_std": 0.625, "grad_norm": 0.832065761089325, "kl": 0.22019675746560097, "learning_rate": 1.501842621639796e-07, "loss": 0.0077, "num_tokens": 18924860.0, "reward": 1.1875, "reward_std": 0.16658930480480194, "rewards/equation_reward_func/mean": 0.1875, "rewards/equation_reward_func/std": 0.39066198468208313, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4917330741882324, "sampling/importance_sampling_ratio/mean": 1.0000223815441132, "sampling/importance_sampling_ratio/min": 0.48453810811042786, "sampling/sampling_logp_difference/max": 0.7312403917312622, "sampling/sampling_logp_difference/mean": 0.006352462340146303, "step": 290 }, { "clip_ratio/high_max": 0.00013053972765596377, "clip_ratio/high_mean": 0.00013053972765596377, "clip_ratio/low_mean": 0.0002916623747700618, "clip_ratio/low_min": 0.0002916623747700618, "clip_ratio/region_mean": 0.0004222021024260256, "completions/clipped_ratio": 0.0, "completions/max_length": 75.0, "completions/max_terminated_length": 75.0, "completions/mean_length": 54.10546875, "completions/mean_terminated_length": 54.10546875, "completions/min_length": 39.0, "completions/min_terminated_length": 39.0, "entropy": 0.12058492480880684, "epoch": 0.07788051209103841, "frac_reward_zero_std": 0.640625, "grad_norm": 1.1296939849853516, "kl": 0.2924755515737666, "learning_rate": 1.4689161405333652e-07, "loss": -0.0042, "num_tokens": 19030778.0, "reward": 1.1640625, "reward_std": 0.16629283130168915, "rewards/equation_reward_func/mean": 0.1640625, "rewards/equation_reward_func/std": 0.37069445848464966, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4458497762680054, "sampling/importance_sampling_ratio/mean": 0.999947726726532, "sampling/importance_sampling_ratio/min": 0.47323113679885864, "sampling/sampling_logp_difference/max": 0.748171329498291, "sampling/sampling_logp_difference/mean": 0.005906645208597183, "step": 292 }, { "clip_ratio/high_max": 0.00015977974787044028, "clip_ratio/high_mean": 0.00015977974787044028, "clip_ratio/low_mean": 0.0003193130736084034, "clip_ratio/low_min": 0.0003193130736084034, "clip_ratio/region_mean": 0.0004790928214788437, "completions/clipped_ratio": 0.0, "completions/max_length": 73.0, "completions/max_terminated_length": 73.0, "completions/mean_length": 52.966796875, "completions/mean_terminated_length": 52.966796875, "completions/min_length": 35.0, "completions/min_terminated_length": 35.0, "entropy": 0.11835546274152067, "epoch": 0.07841394025604552, "frac_reward_zero_std": 0.625, "grad_norm": 0.6429219245910645, "kl": 0.29682688580618966, "learning_rate": 1.4362037873135255e-07, "loss": 0.0076, "num_tokens": 19135753.0, "reward": 1.26171875, "reward_std": 0.171815887093544, "rewards/equation_reward_func/mean": 0.263671875, "rewards/equation_reward_func/std": 0.4410543739795685, "rewards/format_reward_func/mean": 0.998046875, "rewards/format_reward_func/std": 0.04419417306780815, "sampling/importance_sampling_ratio/max": 1.5525758266448975, "sampling/importance_sampling_ratio/mean": 1.000226616859436, "sampling/importance_sampling_ratio/min": 0.5377941727638245, "sampling/sampling_logp_difference/max": 0.6202793121337891, "sampling/sampling_logp_difference/mean": 0.005819721147418022, "step": 294 }, { "clip_ratio/high_max": 0.0001345365469913102, "clip_ratio/high_mean": 0.0001345365469913102, "clip_ratio/low_mean": 0.0001629980009359618, "clip_ratio/low_min": 0.0001629980009359618, "clip_ratio/region_mean": 0.000297534547927272, "completions/clipped_ratio": 0.0, "completions/max_length": 72.0, "completions/max_terminated_length": 72.0, "completions/mean_length": 53.865234375, "completions/mean_terminated_length": 53.865234375, "completions/min_length": 28.0, "completions/min_terminated_length": 28.0, "entropy": 0.11495288068221675, "epoch": 0.07894736842105263, "frac_reward_zero_std": 0.625, "grad_norm": 0.5954465270042419, "kl": 0.24669315500391853, "learning_rate": 1.403712355440378e-07, "loss": -0.0138, "num_tokens": 19241508.0, "reward": 1.134765625, "reward_std": 0.15756143629550934, "rewards/equation_reward_func/mean": 0.134765625, "rewards/equation_reward_func/std": 0.3418070077896118, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4350379705429077, "sampling/importance_sampling_ratio/mean": 0.9998484253883362, "sampling/importance_sampling_ratio/min": 0.4763599932193756, "sampling/sampling_logp_difference/max": 0.7415814399719238, "sampling/sampling_logp_difference/mean": 0.005357466638088226, "step": 296 }, { "clip_ratio/high_max": 0.00012767812909765376, "clip_ratio/high_mean": 0.00012767812909765376, "clip_ratio/low_mean": 0.0001937980479043391, "clip_ratio/low_min": 0.0001937980479043391, "clip_ratio/region_mean": 0.0003214761770019929, "completions/clipped_ratio": 0.0, "completions/max_length": 81.0, "completions/max_terminated_length": 81.0, "completions/mean_length": 54.123046875, "completions/mean_terminated_length": 54.123046875, "completions/min_length": 37.0, "completions/min_terminated_length": 37.0, "entropy": 0.10710652855535348, "epoch": 0.07948079658605975, "frac_reward_zero_std": 0.71875, "grad_norm": 0.5267983675003052, "kl": 0.24504679979549515, "learning_rate": 1.371448592494707e-07, "loss": 0.0166, "num_tokens": 19347403.0, "reward": 1.123046875, "reward_std": 0.13413512706756592, "rewards/equation_reward_func/mean": 0.125, "rewards/equation_reward_func/std": 0.3310423493385315, "rewards/format_reward_func/mean": 0.998046875, "rewards/format_reward_func/std": 0.04419417306780815, "sampling/importance_sampling_ratio/max": 1.4578505754470825, "sampling/importance_sampling_ratio/mean": 0.9998474717140198, "sampling/importance_sampling_ratio/min": 0.6094253659248352, "sampling/sampling_logp_difference/max": 0.4952387809753418, "sampling/sampling_logp_difference/mean": 0.005574563052505255, "step": 298 }, { "clip_ratio/high_max": 9.526715600966579e-05, "clip_ratio/high_mean": 9.526715600966579e-05, "clip_ratio/low_mean": 0.00012599391953295303, "clip_ratio/low_min": 0.00012599391953295303, "clip_ratio/region_mean": 0.00022126107554261884, "completions/clipped_ratio": 0.0, "completions/max_length": 87.0, "completions/max_terminated_length": 87.0, "completions/mean_length": 54.29296875, "completions/mean_terminated_length": 54.29296875, "completions/min_length": 35.0, "completions/min_terminated_length": 35.0, "entropy": 0.10549763672881657, "epoch": 0.08001422475106686, "frac_reward_zero_std": 0.609375, "grad_norm": 0.5649374723434448, "kl": 0.20756946214371258, "learning_rate": 1.3394191987766996e-07, "loss": -0.0085, "num_tokens": 19453401.0, "reward": 1.154296875, "reward_std": 0.17846155166625977, "rewards/equation_reward_func/mean": 0.154296875, "rewards/equation_reward_func/std": 0.36158639192581177, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.9475939273834229, "sampling/importance_sampling_ratio/mean": 0.999893069267273, "sampling/importance_sampling_ratio/min": 0.6712517738342285, "sampling/sampling_logp_difference/max": 0.6665947437286377, "sampling/sampling_logp_difference/mean": 0.005600410513579845, "step": 300 }, { "clip_ratio/high_max": 0.00016323415265004669, "clip_ratio/high_mean": 0.00016323415265004669, "clip_ratio/low_mean": 0.0002246858881941686, "clip_ratio/low_min": 0.0002246858881941686, "clip_ratio/region_mean": 0.0003879200408442153, "completions/clipped_ratio": 0.0, "completions/max_length": 71.0, "completions/max_terminated_length": 71.0, "completions/mean_length": 53.71875, "completions/mean_terminated_length": 53.71875, "completions/min_length": 37.0, "completions/min_terminated_length": 37.0, "entropy": 0.10518266074359417, "epoch": 0.08054765291607396, "frac_reward_zero_std": 0.703125, "grad_norm": 0.6784906387329102, "kl": 0.483879576333695, "learning_rate": 1.3076308259144652e-07, "loss": 0.0048, "num_tokens": 19558953.0, "reward": 1.1796875, "reward_std": 0.12743577361106873, "rewards/equation_reward_func/mean": 0.1796875, "rewards/equation_reward_func/std": 0.38430243730545044, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.575277328491211, "sampling/importance_sampling_ratio/mean": 0.9999927878379822, "sampling/importance_sampling_ratio/min": 0.5361084342002869, "sampling/sampling_logp_difference/max": 0.6234188079833984, "sampling/sampling_logp_difference/mean": 0.005135941784828901, "step": 302 }, { "clip_ratio/high_max": 0.00013082856700445214, "clip_ratio/high_mean": 0.00013082856700445214, "clip_ratio/low_mean": 0.00040843233970614773, "clip_ratio/low_min": 0.00040843233970614773, "clip_ratio/region_mean": 0.0005392609067105999, "completions/clipped_ratio": 0.0, "completions/max_length": 76.0, "completions/max_terminated_length": 76.0, "completions/mean_length": 53.46875, "completions/mean_terminated_length": 53.46875, "completions/min_length": 35.0, "completions/min_terminated_length": 35.0, "entropy": 0.105962166769637, "epoch": 0.08108108108108109, "frac_reward_zero_std": 0.640625, "grad_norm": 0.5560368299484253, "kl": 0.23596913264029556, "learning_rate": 1.2760900754826858e-07, "loss": -0.0004, "num_tokens": 19664321.0, "reward": 1.1953125, "reward_std": 0.1626114547252655, "rewards/equation_reward_func/mean": 0.197265625, "rewards/equation_reward_func/std": 0.3983237147331238, "rewards/format_reward_func/mean": 0.998046875, "rewards/format_reward_func/std": 0.04419417306780815, "sampling/importance_sampling_ratio/max": 1.5302678346633911, "sampling/importance_sampling_ratio/mean": 0.999779224395752, "sampling/importance_sampling_ratio/min": 0.3679559528827667, "sampling/sampling_logp_difference/max": 0.9997920989990234, "sampling/sampling_logp_difference/mean": 0.005438545718789101, "step": 304 }, { "clip_ratio/high_max": 3.4083162770710056e-05, "clip_ratio/high_mean": 3.4083162770710056e-05, "clip_ratio/low_mean": 0.0002949693912847175, "clip_ratio/low_min": 0.0002949693912847175, "clip_ratio/region_mean": 0.0003290525540554275, "completions/clipped_ratio": 0.0, "completions/max_length": 80.5, "completions/max_terminated_length": 80.5, "completions/mean_length": 53.685546875, "completions/mean_terminated_length": 53.685546875, "completions/min_length": 36.0, "completions/min_terminated_length": 36.0, "entropy": 0.10773880945311652, "epoch": 0.0816145092460882, "frac_reward_zero_std": 0.6875, "grad_norm": 0.6665006875991821, "kl": 0.3991919548975097, "learning_rate": 1.2448034976316394e-07, "loss": 0.0043, "num_tokens": 19875319.0, "reward": 1.171875, "reward_std": 0.13664264231920242, "rewards/equation_reward_func/mean": 0.171875, "rewards/equation_reward_func/std": 0.37562617659568787, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.5477519631385803, "sampling/importance_sampling_ratio/mean": 1.0001848340034485, "sampling/importance_sampling_ratio/min": 0.5895287990570068, "sampling/sampling_logp_difference/max": 0.5308138132095337, "sampling/sampling_logp_difference/mean": 0.005096454173326492, "step": 306 }, { "clip_ratio/high_max": 6.455278425063525e-05, "clip_ratio/high_mean": 6.455278425063525e-05, "clip_ratio/low_mean": 0.0001954515884992563, "clip_ratio/low_min": 0.0001954515884992563, "clip_ratio/region_mean": 0.00026000437274989154, "completions/clipped_ratio": 0.0, "completions/max_length": 76.0, "completions/max_terminated_length": 76.0, "completions/mean_length": 53.947265625, "completions/mean_terminated_length": 53.947265625, "completions/min_length": 37.0, "completions/min_terminated_length": 37.0, "entropy": 0.10061582881543371, "epoch": 0.0821479374110953, "frac_reward_zero_std": 0.671875, "grad_norm": 0.5578693151473999, "kl": 0.26390944690340096, "learning_rate": 1.213777589726922e-07, "loss": 0.0015, "num_tokens": 19981004.0, "reward": 1.162109375, "reward_std": 0.1520920991897583, "rewards/equation_reward_func/mean": 0.162109375, "rewards/equation_reward_func/std": 0.3689115643501282, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.6302008628845215, "sampling/importance_sampling_ratio/mean": 0.9996868371963501, "sampling/importance_sampling_ratio/min": 0.47551459074020386, "sampling/sampling_logp_difference/max": 0.7433576583862305, "sampling/sampling_logp_difference/mean": 0.005319277290254831, "step": 308 }, { "clip_ratio/high_max": 0.00013303185632038448, "clip_ratio/high_mean": 0.00013303185632038448, "clip_ratio/low_mean": 9.891274385154247e-05, "clip_ratio/low_min": 9.891274385154247e-05, "clip_ratio/region_mean": 0.00023194460017192695, "completions/clipped_ratio": 0.0, "completions/max_length": 115.0, "completions/max_terminated_length": 115.0, "completions/mean_length": 54.548828125, "completions/mean_terminated_length": 54.548828125, "completions/min_length": 38.0, "completions/min_terminated_length": 38.0, "entropy": 0.10588591939045323, "epoch": 0.08268136557610242, "frac_reward_zero_std": 0.625, "grad_norm": 0.7521274089813232, "kl": 0.27938652514583534, "learning_rate": 1.183018795000118e-07, "loss": -0.0094, "num_tokens": 20087141.0, "reward": 1.201171875, "reward_std": 0.16781118512153625, "rewards/equation_reward_func/mean": 0.201171875, "rewards/equation_reward_func/std": 0.4012683033943176, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.5131198167800903, "sampling/importance_sampling_ratio/mean": 0.999969482421875, "sampling/importance_sampling_ratio/min": 0.5128318667411804, "sampling/sampling_logp_difference/max": 0.6678072214126587, "sampling/sampling_logp_difference/mean": 0.005433302838355303, "step": 310 }, { "clip_ratio/high_max": 9.650004600795607e-05, "clip_ratio/high_mean": 9.650004600795607e-05, "clip_ratio/low_mean": 0.0002817239728756249, "clip_ratio/low_min": 0.0002817239728756249, "clip_ratio/region_mean": 0.000378224018883581, "completions/clipped_ratio": 0.0, "completions/max_length": 76.0, "completions/max_terminated_length": 76.0, "completions/mean_length": 54.435546875, "completions/mean_terminated_length": 54.435546875, "completions/min_length": 34.0, "completions/min_terminated_length": 34.0, "entropy": 0.10187006679673989, "epoch": 0.08321479374110953, "frac_reward_zero_std": 0.75, "grad_norm": 0.49840039014816284, "kl": 0.21569028579526478, "learning_rate": 1.1525335012107188e-07, "loss": 0.002, "num_tokens": 20193292.0, "reward": 1.076171875, "reward_std": 0.10514955222606659, "rewards/equation_reward_func/mean": 0.076171875, "rewards/equation_reward_func/std": 0.26553234457969666, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.6213353872299194, "sampling/importance_sampling_ratio/mean": 0.999853789806366, "sampling/importance_sampling_ratio/min": 0.5379842519760132, "sampling/sampling_logp_difference/max": 0.6199259757995605, "sampling/sampling_logp_difference/mean": 0.00533775519579649, "step": 312 }, { "clip_ratio/high_max": 0.00012695331436892351, "clip_ratio/high_mean": 0.00012695331436892351, "clip_ratio/low_mean": 0.00022922881503796412, "clip_ratio/low_min": 0.00022922881503796412, "clip_ratio/region_mean": 0.00035618212940688763, "completions/clipped_ratio": 0.0, "completions/max_length": 76.0, "completions/max_terminated_length": 76.0, "completions/mean_length": 53.759765625, "completions/mean_terminated_length": 53.759765625, "completions/min_length": 38.0, "completions/min_terminated_length": 38.0, "entropy": 0.09872452769842413, "epoch": 0.08374822190611664, "frac_reward_zero_std": 0.671875, "grad_norm": 0.8471410870552063, "kl": 0.24236748119195303, "learning_rate": 1.1223280393195566e-07, "loss": 0.0079, "num_tokens": 20298905.0, "reward": 1.138671875, "reward_std": 0.14611473679542542, "rewards/equation_reward_func/mean": 0.138671875, "rewards/equation_reward_func/std": 0.34594178199768066, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.8655023574829102, "sampling/importance_sampling_ratio/mean": 1.0001434087753296, "sampling/importance_sampling_ratio/min": 0.6093878746032715, "sampling/sampling_logp_difference/max": 0.623530387878418, "sampling/sampling_logp_difference/mean": 0.0052959248423576355, "step": 314 }, { "clip_ratio/high_max": 0.00019498604039351145, "clip_ratio/high_mean": 0.00019498604039351145, "clip_ratio/low_mean": 0.00022457260961851312, "clip_ratio/low_min": 0.00022457260961851312, "clip_ratio/region_mean": 0.00041955865001202456, "completions/clipped_ratio": 0.0, "completions/max_length": 78.0, "completions/max_terminated_length": 78.0, "completions/mean_length": 52.978515625, "completions/mean_terminated_length": 52.978515625, "completions/min_length": 38.0, "completions/min_terminated_length": 38.0, "entropy": 0.09926377278235224, "epoch": 0.08428165007112376, "frac_reward_zero_std": 0.71875, "grad_norm": 0.5593821406364441, "kl": 0.26706351588169736, "learning_rate": 1.0924086821740436e-07, "loss": -0.0012, "num_tokens": 20403910.0, "reward": 1.158203125, "reward_std": 0.12848231196403503, "rewards/equation_reward_func/mean": 0.158203125, "rewards/equation_reward_func/std": 0.36528825759887695, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.3883647918701172, "sampling/importance_sampling_ratio/mean": 0.9999933242797852, "sampling/importance_sampling_ratio/min": 0.5356259942054749, "sampling/sampling_logp_difference/max": 0.6243190765380859, "sampling/sampling_logp_difference/mean": 0.004806127864867449, "step": 316 }, { "clip_ratio/high_max": 0.00019597292864798673, "clip_ratio/high_mean": 0.00019597292864798673, "clip_ratio/low_mean": 0.00016192727748097645, "clip_ratio/low_min": 0.00016192727748097645, "clip_ratio/region_mean": 0.00035790020612896315, "completions/clipped_ratio": 0.0, "completions/max_length": 74.0, "completions/max_terminated_length": 74.0, "completions/mean_length": 53.642578125, "completions/mean_terminated_length": 53.642578125, "completions/min_length": 35.0, "completions/min_terminated_length": 35.0, "entropy": 0.10184383454422156, "epoch": 0.08481507823613087, "frac_reward_zero_std": 0.65625, "grad_norm": 0.5862502455711365, "kl": 0.27017481583687997, "learning_rate": 1.0627816432054689e-07, "loss": -0.004, "num_tokens": 20509407.0, "reward": 1.177734375, "reward_std": 0.1521681845188141, "rewards/equation_reward_func/mean": 0.1796875, "rewards/equation_reward_func/std": 0.38430243730545044, "rewards/format_reward_func/mean": 0.998046875, "rewards/format_reward_func/std": 0.04419417306780815, "sampling/importance_sampling_ratio/max": 1.8215539455413818, "sampling/importance_sampling_ratio/mean": 1.0000892877578735, "sampling/importance_sampling_ratio/min": 0.5373824834823608, "sampling/sampling_logp_difference/max": 0.6210451126098633, "sampling/sampling_logp_difference/mean": 0.004881645552814007, "step": 318 }, { "clip_ratio/high_max": 0.00016617278258005777, "clip_ratio/high_mean": 0.00016617278258005777, "clip_ratio/low_mean": 3.062599636096921e-05, "clip_ratio/low_min": 3.062599636096921e-05, "clip_ratio/region_mean": 0.000196798778941027, "completions/clipped_ratio": 0.0, "completions/max_length": 74.0, "completions/max_terminated_length": 74.0, "completions/mean_length": 53.58984375, "completions/mean_terminated_length": 53.58984375, "completions/min_length": 38.0, "completions/min_terminated_length": 38.0, "entropy": 0.09652745206322935, "epoch": 0.08534850640113797, "frac_reward_zero_std": 0.703125, "grad_norm": 0.7487227916717529, "kl": 0.30668989858693546, "learning_rate": 1.0334530751386386e-07, "loss": 0.0024, "num_tokens": 20614861.0, "reward": 1.14453125, "reward_std": 0.13058918714523315, "rewards/equation_reward_func/mean": 0.14453125, "rewards/equation_reward_func/std": 0.35197147727012634, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.9990084171295166, "sampling/importance_sampling_ratio/mean": 0.9999464154243469, "sampling/importance_sampling_ratio/min": 0.47329431772232056, "sampling/sampling_logp_difference/max": 0.7480378150939941, "sampling/sampling_logp_difference/mean": 0.004811522550880909, "step": 320 }, { "clip_ratio/high_max": 2.890507716478573e-05, "clip_ratio/high_mean": 2.890507716478573e-05, "clip_ratio/low_mean": 0.00012445226376358833, "clip_ratio/low_min": 0.00012445226376358833, "clip_ratio/region_mean": 0.00015335734092837406, "completions/clipped_ratio": 0.0, "completions/max_length": 85.0, "completions/max_terminated_length": 85.0, "completions/mean_length": 54.3740234375, "completions/mean_terminated_length": 54.3740234375, "completions/min_length": 38.0, "completions/min_terminated_length": 38.0, "entropy": 0.09888760331604216, "epoch": 0.0858819345661451, "frac_reward_zero_std": 0.71875, "grad_norm": 0.5923455357551575, "kl": 0.288042470605837, "learning_rate": 1.0044290687141255e-07, "loss": -0.0038, "num_tokens": 20826892.0, "reward": 1.1435546875, "reward_std": 0.12148462608456612, "rewards/equation_reward_func/mean": 0.1435546875, "rewards/equation_reward_func/std": 0.34574414789676666, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.589076280593872, "sampling/importance_sampling_ratio/mean": 1.000116765499115, "sampling/importance_sampling_ratio/min": 0.34139294922351837, "sampling/sampling_logp_difference/max": 1.1013591885566711, "sampling/sampling_logp_difference/mean": 0.004871859215199947, "step": 322 }, { "clip_ratio/high_max": 0.00016105886445277266, "clip_ratio/high_mean": 0.00016105886445277266, "clip_ratio/low_mean": 0.0001920885891498377, "clip_ratio/low_min": 0.0001920885891498377, "clip_ratio/region_mean": 0.0003531474536026104, "completions/clipped_ratio": 0.0, "completions/max_length": 76.0, "completions/max_terminated_length": 76.0, "completions/mean_length": 54.263671875, "completions/mean_terminated_length": 54.263671875, "completions/min_length": 38.0, "completions/min_terminated_length": 38.0, "entropy": 0.09685155438880126, "epoch": 0.0864153627311522, "frac_reward_zero_std": 0.6875, "grad_norm": 0.4126549959182739, "kl": 0.2570730716817909, "learning_rate": 9.757156514233892e-08, "loss": 0.0147, "num_tokens": 20932923.0, "reward": 1.126953125, "reward_std": 0.13776035606861115, "rewards/equation_reward_func/mean": 0.126953125, "rewards/equation_reward_func/std": 0.33324605226516724, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.5707911252975464, "sampling/importance_sampling_ratio/mean": 1.0002734661102295, "sampling/importance_sampling_ratio/min": 0.4189390242099762, "sampling/sampling_logp_difference/max": 0.8700299263000488, "sampling/sampling_logp_difference/mean": 0.005099819041788578, "step": 324 }, { "clip_ratio/high_max": 0.00012838508700951934, "clip_ratio/high_mean": 0.00012838508700951934, "clip_ratio/low_mean": 9.717278104896347e-05, "clip_ratio/low_min": 9.717278104896347e-05, "clip_ratio/region_mean": 0.0002255578680584828, "completions/clipped_ratio": 0.0, "completions/max_length": 69.0, "completions/max_terminated_length": 69.0, "completions/mean_length": 53.513671875, "completions/mean_terminated_length": 53.513671875, "completions/min_length": 35.0, "completions/min_terminated_length": 35.0, "entropy": 0.09719797016845809, "epoch": 0.08694879089615931, "frac_reward_zero_std": 0.6875, "grad_norm": 0.6993542909622192, "kl": 0.22395568154752254, "learning_rate": 9.473187862570289e-08, "loss": -0.0121, "num_tokens": 21038298.0, "reward": 1.20703125, "reward_std": 0.13973549008369446, "rewards/equation_reward_func/mean": 0.20703125, "rewards/equation_reward_func/std": 0.40557438135147095, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.5042448043823242, "sampling/importance_sampling_ratio/mean": 1.0001046657562256, "sampling/importance_sampling_ratio/min": 0.5376067757606506, "sampling/sampling_logp_difference/max": 0.6206278800964355, "sampling/sampling_logp_difference/mean": 0.004805511329323053, "step": 326 }, { "clip_ratio/high_max": 0.00013057675096206367, "clip_ratio/high_mean": 0.00013057675096206367, "clip_ratio/low_mean": 6.622885444408489e-05, "clip_ratio/low_min": 6.622885444408489e-05, "clip_ratio/region_mean": 0.00019680560540614856, "completions/clipped_ratio": 0.0, "completions/max_length": 72.0, "completions/max_terminated_length": 72.0, "completions/mean_length": 53.38671875, "completions/mean_terminated_length": 53.38671875, "completions/min_length": 34.0, "completions/min_terminated_length": 34.0, "entropy": 0.09837449114355776, "epoch": 0.08748221906116643, "frac_reward_zero_std": 0.671875, "grad_norm": 0.7079157829284668, "kl": 0.1822098141743077, "learning_rate": 9.192443704664344e-08, "loss": 0.0066, "num_tokens": 21143592.0, "reward": 1.18359375, "reward_std": 0.15097439289093018, "rewards/equation_reward_func/mean": 0.185546875, "rewards/equation_reward_func/std": 0.38912075757980347, "rewards/format_reward_func/mean": 0.998046875, "rewards/format_reward_func/std": 0.04419417306780815, "sampling/importance_sampling_ratio/max": 1.5526940822601318, "sampling/importance_sampling_ratio/mean": 1.000152349472046, "sampling/importance_sampling_ratio/min": 0.6077161431312561, "sampling/sampling_logp_difference/max": 0.4980473518371582, "sampling/sampling_logp_difference/mean": 0.00505709508433938, "step": 328 }, { "clip_ratio/high_max": 0.00019603894260298047, "clip_ratio/high_mean": 0.00019603894260298047, "clip_ratio/low_mean": 0.00019100008648820221, "clip_ratio/low_min": 0.00019100008648820221, "clip_ratio/region_mean": 0.0003870390290911827, "completions/clipped_ratio": 0.0, "completions/max_length": 71.0, "completions/max_terminated_length": 71.0, "completions/mean_length": 54.005859375, "completions/mean_terminated_length": 54.005859375, "completions/min_length": 36.0, "completions/min_terminated_length": 36.0, "entropy": 0.10252868135770161, "epoch": 0.08801564722617354, "frac_reward_zero_std": 0.765625, "grad_norm": 0.4996802508831024, "kl": 0.28261983104878, "learning_rate": 8.914982343390895e-08, "loss": -0.0071, "num_tokens": 21249315.0, "reward": 1.08984375, "reward_std": 0.10902006179094315, "rewards/equation_reward_func/mean": 0.08984375, "rewards/equation_reward_func/std": 0.2862374484539032, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.6295589208602905, "sampling/importance_sampling_ratio/mean": 0.9999970197677612, "sampling/importance_sampling_ratio/min": 0.5906658172607422, "sampling/sampling_logp_difference/max": 0.5265048742294312, "sampling/sampling_logp_difference/mean": 0.005076720379292965, "step": 330 }, { "clip_ratio/high_max": 6.288123808594214e-05, "clip_ratio/high_mean": 6.288123808594214e-05, "clip_ratio/low_mean": 0.00012890514658970965, "clip_ratio/low_min": 0.00012890514658970965, "clip_ratio/region_mean": 0.0001917863846756518, "completions/clipped_ratio": 0.0, "completions/max_length": 74.0, "completions/max_terminated_length": 74.0, "completions/mean_length": 53.537109375, "completions/mean_terminated_length": 53.537109375, "completions/min_length": 31.0, "completions/min_terminated_length": 31.0, "entropy": 0.09989326985345946, "epoch": 0.08854907539118065, "frac_reward_zero_std": 0.609375, "grad_norm": 0.7519966959953308, "kl": 0.21807977060476938, "learning_rate": 8.640861399877805e-08, "loss": 0.007, "num_tokens": 21354734.0, "reward": 1.22265625, "reward_std": 0.1729392409324646, "rewards/equation_reward_func/mean": 0.224609375, "rewards/equation_reward_func/std": 0.41773295402526855, "rewards/format_reward_func/mean": 0.998046875, "rewards/format_reward_func/std": 0.04419417306780815, "sampling/importance_sampling_ratio/max": 1.5419955253601074, "sampling/importance_sampling_ratio/mean": 1.0000159740447998, "sampling/importance_sampling_ratio/min": 0.41770172119140625, "sampling/sampling_logp_difference/max": 0.8729877471923828, "sampling/sampling_logp_difference/mean": 0.005091718398034573, "step": 332 }, { "clip_ratio/high_max": 3.233734363069137e-05, "clip_ratio/high_mean": 3.233734363069137e-05, "clip_ratio/low_mean": 0.00038376119401719834, "clip_ratio/low_min": 0.00038376119401719834, "clip_ratio/region_mean": 0.0004160985376478897, "completions/clipped_ratio": 0.0, "completions/max_length": 100.0, "completions/max_terminated_length": 100.0, "completions/mean_length": 54.75390625, "completions/mean_terminated_length": 54.75390625, "completions/min_length": 36.0, "completions/min_terminated_length": 36.0, "entropy": 0.09954109022186862, "epoch": 0.08908250355618777, "frac_reward_zero_std": 0.671875, "grad_norm": 0.6240397691726685, "kl": 0.21472362304727236, "learning_rate": 8.370137801539634e-08, "loss": 0.0042, "num_tokens": 21461064.0, "reward": 1.138671875, "reward_std": 0.14157837629318237, "rewards/equation_reward_func/mean": 0.138671875, "rewards/equation_reward_func/std": 0.34594178199768066, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.5789878368377686, "sampling/importance_sampling_ratio/mean": 1.0002344846725464, "sampling/importance_sampling_ratio/min": 0.4178570806980133, "sampling/sampling_logp_difference/max": 0.8726158142089844, "sampling/sampling_logp_difference/mean": 0.005193983670324087, "step": 334 }, { "clip_ratio/high_max": 9.585934458300471e-05, "clip_ratio/high_mean": 9.585934458300471e-05, "clip_ratio/low_mean": 0.00012715121071475247, "clip_ratio/low_min": 0.00012715121071475247, "clip_ratio/region_mean": 0.00022301055529775718, "completions/clipped_ratio": 0.0, "completions/max_length": 70.0, "completions/max_terminated_length": 70.0, "completions/mean_length": 54.646484375, "completions/mean_terminated_length": 54.646484375, "completions/min_length": 37.0, "completions/min_terminated_length": 37.0, "entropy": 0.10077821670307054, "epoch": 0.08961593172119488, "frac_reward_zero_std": 0.671875, "grad_norm": 0.6173285245895386, "kl": 0.2103740041040712, "learning_rate": 8.102867770255337e-08, "loss": -0.0037, "num_tokens": 21567323.0, "reward": 1.140625, "reward_std": 0.1521514505147934, "rewards/equation_reward_func/mean": 0.140625, "rewards/equation_reward_func/std": 0.3479743003845215, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.6297080516815186, "sampling/importance_sampling_ratio/mean": 1.0000563859939575, "sampling/importance_sampling_ratio/min": 0.5369880199432373, "sampling/sampling_logp_difference/max": 0.6217794418334961, "sampling/sampling_logp_difference/mean": 0.0048846774734556675, "step": 336 }, { "clip_ratio/high_max": 3.306878433148894e-05, "clip_ratio/high_mean": 3.306878433148894e-05, "clip_ratio/low_mean": 6.596661599663396e-05, "clip_ratio/low_min": 6.596661599663396e-05, "clip_ratio/region_mean": 9.903540032812291e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 78.0, "completions/max_terminated_length": 78.0, "completions/mean_length": 53.82421875, "completions/mean_terminated_length": 53.82421875, "completions/min_length": 35.0, "completions/min_terminated_length": 35.0, "entropy": 0.09935227346916993, "epoch": 0.09014935988620199, "frac_reward_zero_std": 0.6875, "grad_norm": 0.6280437707901001, "kl": 0.2277641739282343, "learning_rate": 7.839106810692589e-08, "loss": -0.0009, "num_tokens": 21778639.0, "reward": 1.1806640625, "reward_std": 0.13480134308338165, "rewards/equation_reward_func/mean": 0.1806640625, "rewards/equation_reward_func/std": 0.3841889351606369, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.7198472619056702, "sampling/importance_sampling_ratio/mean": 0.9999500811100006, "sampling/importance_sampling_ratio/min": 0.6093293130397797, "sampling/sampling_logp_difference/max": 0.613457202911377, "sampling/sampling_logp_difference/mean": 0.004906615940853953, "step": 338 }, { "clip_ratio/high_max": 9.600156206741101e-05, "clip_ratio/high_mean": 9.600156206741101e-05, "clip_ratio/low_mean": 0.00012920183427114453, "clip_ratio/low_min": 0.00012920183427114453, "clip_ratio/region_mean": 0.00022520339633855555, "completions/clipped_ratio": 0.0, "completions/max_length": 77.0, "completions/max_terminated_length": 77.0, "completions/mean_length": 54.6484375, "completions/mean_terminated_length": 54.6484375, "completions/min_length": 34.0, "completions/min_terminated_length": 34.0, "entropy": 0.09617070563965374, "epoch": 0.0906827880512091, "frac_reward_zero_std": 0.6875, "grad_norm": 0.7513833045959473, "kl": 0.42414102144539356, "learning_rate": 7.57890969878093e-08, "loss": -0.0044, "num_tokens": 21884891.0, "reward": 1.21875, "reward_std": 0.14459636807441711, "rewards/equation_reward_func/mean": 0.220703125, "rewards/equation_reward_func/std": 0.4151262938976288, "rewards/format_reward_func/mean": 0.998046875, "rewards/format_reward_func/std": 0.04419417306780815, "sampling/importance_sampling_ratio/max": 1.6317570209503174, "sampling/importance_sampling_ratio/mean": 1.0000431537628174, "sampling/importance_sampling_ratio/min": 0.6167507171630859, "sampling/sampling_logp_difference/max": 0.4896574020385742, "sampling/sampling_logp_difference/mean": 0.004871396813541651, "step": 340 }, { "clip_ratio/high_max": 0.00012448259642244212, "clip_ratio/high_mean": 0.00012448259642244212, "clip_ratio/low_mean": 9.562747759951485e-05, "clip_ratio/low_min": 9.562747759951485e-05, "clip_ratio/region_mean": 0.000220110074021957, "completions/clipped_ratio": 0.0, "completions/max_length": 70.0, "completions/max_terminated_length": 70.0, "completions/mean_length": 53.703125, "completions/mean_terminated_length": 53.703125, "completions/min_length": 34.0, "completions/min_terminated_length": 34.0, "entropy": 0.0994571488764551, "epoch": 0.09121621621621621, "frac_reward_zero_std": 0.6875, "grad_norm": 0.6678158044815063, "kl": 0.5161584214203887, "learning_rate": 7.322330470336313e-08, "loss": 0.0061, "num_tokens": 21990387.0, "reward": 1.1328125, "reward_std": 0.12927988171577454, "rewards/equation_reward_func/mean": 0.1328125, "rewards/equation_reward_func/std": 0.33970388770103455, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4448696374893188, "sampling/importance_sampling_ratio/mean": 0.9999097585678101, "sampling/importance_sampling_ratio/min": 0.47708994150161743, "sampling/sampling_logp_difference/max": 0.7400503158569336, "sampling/sampling_logp_difference/mean": 0.005025411024689674, "step": 342 }, { "clip_ratio/high_max": 6.34941356515305e-05, "clip_ratio/high_mean": 6.34941356515305e-05, "clip_ratio/low_mean": 9.748389452902807e-05, "clip_ratio/low_min": 9.748389452902807e-05, "clip_ratio/region_mean": 0.00016097803018055856, "completions/clipped_ratio": 0.0, "completions/max_length": 80.0, "completions/max_terminated_length": 80.0, "completions/mean_length": 54.388671875, "completions/mean_terminated_length": 54.388671875, "completions/min_length": 36.0, "completions/min_terminated_length": 36.0, "entropy": 0.10490797604951593, "epoch": 0.09174964438122332, "frac_reward_zero_std": 0.609375, "grad_norm": 0.5954409241676331, "kl": 0.23606261528200573, "learning_rate": 7.069422409839363e-08, "loss": 0.0057, "num_tokens": 22096346.0, "reward": 1.205078125, "reward_std": 0.17872552573680878, "rewards/equation_reward_func/mean": 0.205078125, "rewards/equation_reward_func/std": 0.4041535556316376, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4162734746932983, "sampling/importance_sampling_ratio/mean": 1.0001779794692993, "sampling/importance_sampling_ratio/min": 0.5386984348297119, "sampling/sampling_logp_difference/max": 0.6185994148254395, "sampling/sampling_logp_difference/mean": 0.005469147115945816, "step": 344 }, { "clip_ratio/high_max": 0.0001925600388656474, "clip_ratio/high_mean": 0.0001925600388656474, "clip_ratio/low_mean": 0.0002929596521425992, "clip_ratio/low_min": 0.0002929596521425992, "clip_ratio/region_mean": 0.0004855196910082466, "completions/clipped_ratio": 0.0, "completions/max_length": 76.0, "completions/max_terminated_length": 76.0, "completions/mean_length": 53.69921875, "completions/mean_terminated_length": 53.69921875, "completions/min_length": 44.0, "completions/min_terminated_length": 44.0, "entropy": 0.10846325796511438, "epoch": 0.09228307254623044, "frac_reward_zero_std": 0.578125, "grad_norm": 0.5828835964202881, "kl": 0.21855774811572498, "learning_rate": 6.820238039369647e-08, "loss": -0.0111, "num_tokens": 22201808.0, "reward": 1.208984375, "reward_std": 0.1736050695180893, "rewards/equation_reward_func/mean": 0.208984375, "rewards/equation_reward_func/std": 0.40698084235191345, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.6369379758834839, "sampling/importance_sampling_ratio/mean": 0.9999135732650757, "sampling/importance_sampling_ratio/min": 0.25863245129585266, "sampling/sampling_logp_difference/max": 1.3523473739624023, "sampling/sampling_logp_difference/mean": 0.0053120688535273075, "step": 346 }, { "clip_ratio/high_max": 0.00012726225152922174, "clip_ratio/high_mean": 0.00012726225152922174, "clip_ratio/low_mean": 0.0001607260104113569, "clip_ratio/low_min": 0.0001607260104113569, "clip_ratio/region_mean": 0.00028798826194057864, "completions/clipped_ratio": 0.0, "completions/max_length": 76.0, "completions/max_terminated_length": 76.0, "completions/mean_length": 54.6484375, "completions/mean_terminated_length": 54.6484375, "completions/min_length": 38.0, "completions/min_terminated_length": 38.0, "entropy": 0.10222871187660429, "epoch": 0.09281650071123755, "frac_reward_zero_std": 0.625, "grad_norm": 0.6385054588317871, "kl": 0.20221902802586555, "learning_rate": 6.574829107698238e-08, "loss": 0.0178, "num_tokens": 22308028.0, "reward": 1.22265625, "reward_std": 0.16951988637447357, "rewards/equation_reward_func/mean": 0.22265625, "rewards/equation_reward_func/std": 0.41643625497817993, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.8017233610153198, "sampling/importance_sampling_ratio/mean": 0.9999393820762634, "sampling/importance_sampling_ratio/min": 0.6400265693664551, "sampling/sampling_logp_difference/max": 0.5887436866760254, "sampling/sampling_logp_difference/mean": 0.005080735310912132, "step": 348 }, { "clip_ratio/high_max": 0.00012558145681396127, "clip_ratio/high_mean": 0.00012558145681396127, "clip_ratio/low_mean": 6.365163588068551e-05, "clip_ratio/low_min": 6.365163588068551e-05, "clip_ratio/region_mean": 0.0001892330926946468, "completions/clipped_ratio": 0.0, "completions/max_length": 77.0, "completions/max_terminated_length": 77.0, "completions/mean_length": 53.412109375, "completions/mean_terminated_length": 53.412109375, "completions/min_length": 38.0, "completions/min_terminated_length": 38.0, "entropy": 0.10489507785273923, "epoch": 0.09334992887624466, "frac_reward_zero_std": 0.796875, "grad_norm": 0.49755725264549255, "kl": 0.2944268166191048, "learning_rate": 6.333246579540971e-08, "loss": -0.0153, "num_tokens": 22413359.0, "reward": 1.13671875, "reward_std": 0.0862782895565033, "rewards/equation_reward_func/mean": 0.13671875, "rewards/equation_reward_func/std": 0.3438861668109894, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.7254022359848022, "sampling/importance_sampling_ratio/mean": 1.000238060951233, "sampling/importance_sampling_ratio/min": 0.6082823872566223, "sampling/sampling_logp_difference/max": 0.5454602241516113, "sampling/sampling_logp_difference/mean": 0.0053401063196361065, "step": 350 }, { "clip_ratio/high_max": 6.313139520999457e-05, "clip_ratio/high_mean": 6.313139520999457e-05, "clip_ratio/low_mean": 9.715992038966053e-05, "clip_ratio/low_min": 9.715992038966053e-05, "clip_ratio/region_mean": 0.00016029131559965512, "completions/clipped_ratio": 0.0, "completions/max_length": 71.0, "completions/max_terminated_length": 71.0, "completions/mean_length": 53.89453125, "completions/mean_terminated_length": 53.89453125, "completions/min_length": 36.0, "completions/min_terminated_length": 36.0, "entropy": 0.10171776244209872, "epoch": 0.09388335704125178, "frac_reward_zero_std": 0.734375, "grad_norm": 0.5731579661369324, "kl": 0.2143559284094307, "learning_rate": 6.095540624974435e-08, "loss": 0.0051, "num_tokens": 22519017.0, "reward": 1.115234375, "reward_std": 0.11099714785814285, "rewards/equation_reward_func/mean": 0.115234375, "rewards/equation_reward_func/std": 0.3196168541908264, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.3801380395889282, "sampling/importance_sampling_ratio/mean": 0.9999234080314636, "sampling/importance_sampling_ratio/min": 0.47313907742500305, "sampling/sampling_logp_difference/max": 0.7483658790588379, "sampling/sampling_logp_difference/mean": 0.00496212812140584, "step": 352 }, { "clip_ratio/high_max": 6.536028942921096e-05, "clip_ratio/high_mean": 6.536028942921096e-05, "clip_ratio/low_mean": 9.709978540841903e-05, "clip_ratio/low_min": 9.709978540841903e-05, "clip_ratio/region_mean": 0.00016246007483763, "completions/clipped_ratio": 0.0, "completions/max_length": 77.0, "completions/max_terminated_length": 77.0, "completions/mean_length": 54.0927734375, "completions/mean_terminated_length": 54.0927734375, "completions/min_length": 36.0, "completions/min_terminated_length": 36.0, "entropy": 0.10690712990860145, "epoch": 0.09441678520625889, "frac_reward_zero_std": 0.671875, "grad_norm": 0.5846546292304993, "kl": 0.2801658991310332, "learning_rate": 5.861760609017002e-08, "loss": 0.0067, "num_tokens": 22730616.0, "reward": 1.181640625, "reward_std": 0.14229890704154968, "rewards/equation_reward_func/mean": 0.1826171875, "rewards/equation_reward_func/std": 0.38598138093948364, "rewards/format_reward_func/mean": 0.9990234375, "rewards/format_reward_func/std": 0.022097086533904076, "sampling/importance_sampling_ratio/max": 1.4574605226516724, "sampling/importance_sampling_ratio/mean": 0.9999331831932068, "sampling/importance_sampling_ratio/min": 0.5064413398504257, "sampling/sampling_logp_difference/max": 0.6823024749755859, "sampling/sampling_logp_difference/mean": 0.00533159333281219, "step": 354 }, { "clip_ratio/high_max": 0.00028676756644724973, "clip_ratio/high_mean": 0.00028676756644724973, "clip_ratio/low_mean": 0.0001584082298601667, "clip_ratio/low_min": 0.0001584082298601667, "clip_ratio/region_mean": 0.0004451757963074164, "completions/clipped_ratio": 0.0, "completions/max_length": 80.0, "completions/max_terminated_length": 80.0, "completions/mean_length": 53.71484375, "completions/mean_terminated_length": 53.71484375, "completions/min_length": 42.0, "completions/min_terminated_length": 42.0, "entropy": 0.10310176987614897, "epoch": 0.09495021337126601, "frac_reward_zero_std": 0.765625, "grad_norm": 0.6154847145080566, "kl": 0.20601044533153376, "learning_rate": 5.63195508137711e-08, "loss": -0.0053, "num_tokens": 22836158.0, "reward": 1.15625, "reward_std": 0.10955046862363815, "rewards/equation_reward_func/mean": 0.15625, "rewards/equation_reward_func/std": 0.36344730854034424, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.6115760803222656, "sampling/importance_sampling_ratio/mean": 0.9999242424964905, "sampling/importance_sampling_ratio/min": 0.5363020300865173, "sampling/sampling_logp_difference/max": 0.6230578422546387, "sampling/sampling_logp_difference/mean": 0.0054749916307628155, "step": 356 }, { "clip_ratio/high_max": 6.517285105979277e-05, "clip_ratio/high_mean": 6.517285105979277e-05, "clip_ratio/low_mean": 3.2075955661841566e-05, "clip_ratio/low_min": 3.2075955661841566e-05, "clip_ratio/region_mean": 9.724880672163433e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 74.0, "completions/max_terminated_length": 74.0, "completions/mean_length": 54.1484375, "completions/mean_terminated_length": 54.1484375, "completions/min_length": 41.0, "completions/min_terminated_length": 41.0, "entropy": 0.10751371085643768, "epoch": 0.09548364153627312, "frac_reward_zero_std": 0.8125, "grad_norm": 0.5332208871841431, "kl": 0.32863597178624737, "learning_rate": 5.4061717663707843e-08, "loss": -0.0087, "num_tokens": 22942002.0, "reward": 1.095703125, "reward_std": 0.07365524023771286, "rewards/equation_reward_func/mean": 0.095703125, "rewards/equation_reward_func/std": 0.2944713830947876, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.6140475273132324, "sampling/importance_sampling_ratio/mean": 1.0001451969146729, "sampling/importance_sampling_ratio/min": 0.5362913012504578, "sampling/sampling_logp_difference/max": 0.6230778694152832, "sampling/sampling_logp_difference/mean": 0.005398309323936701, "step": 358 }, { "clip_ratio/high_max": 3.128128203873833e-05, "clip_ratio/high_mean": 3.128128203873833e-05, "clip_ratio/low_mean": 6.333636378662454e-05, "clip_ratio/low_min": 6.333636378662454e-05, "clip_ratio/region_mean": 9.461764582536287e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 76.0, "completions/max_terminated_length": 76.0, "completions/mean_length": 53.45703125, "completions/mean_terminated_length": 53.45703125, "completions/min_length": 38.0, "completions/min_terminated_length": 38.0, "entropy": 0.10722704066170587, "epoch": 0.09601706970128022, "frac_reward_zero_std": 0.65625, "grad_norm": 0.6317431330680847, "kl": 0.28589126840233803, "learning_rate": 5.1844575530106265e-08, "loss": 0.0084, "num_tokens": 23047372.0, "reward": 1.16796875, "reward_std": 0.15452352166175842, "rewards/equation_reward_func/mean": 0.169921875, "rewards/equation_reward_func/std": 0.3759314715862274, "rewards/format_reward_func/mean": 0.998046875, "rewards/format_reward_func/std": 0.04419417306780815, "sampling/importance_sampling_ratio/max": 1.5585031509399414, "sampling/importance_sampling_ratio/mean": 1.0002102851867676, "sampling/importance_sampling_ratio/min": 0.37018513679504395, "sampling/sampling_logp_difference/max": 0.9937520027160645, "sampling/sampling_logp_difference/mean": 0.005061218049377203, "step": 360 }, { "clip_ratio/high_max": 9.611789654526446e-05, "clip_ratio/high_mean": 9.611789654526446e-05, "clip_ratio/low_mean": 6.339250184181664e-05, "clip_ratio/low_min": 6.339250184181664e-05, "clip_ratio/region_mean": 0.0001595103983870811, "completions/clipped_ratio": 0.0, "completions/max_length": 71.0, "completions/max_terminated_length": 71.0, "completions/mean_length": 54.3828125, "completions/mean_terminated_length": 54.3828125, "completions/min_length": 33.0, "completions/min_terminated_length": 33.0, "entropy": 0.11144272796809673, "epoch": 0.09655049786628735, "frac_reward_zero_std": 0.671875, "grad_norm": 0.6381310224533081, "kl": 0.47150007800923455, "learning_rate": 4.9668584852682134e-08, "loss": 0.0046, "num_tokens": 23153472.0, "reward": 1.146484375, "reward_std": 0.14584828913211823, "rewards/equation_reward_func/mean": 0.146484375, "rewards/equation_reward_func/std": 0.35393697023391724, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4582003355026245, "sampling/importance_sampling_ratio/mean": 1.000150442123413, "sampling/importance_sampling_ratio/min": 0.5402125120162964, "sampling/sampling_logp_difference/max": 0.6157927513122559, "sampling/sampling_logp_difference/mean": 0.005494322627782822, "step": 362 }, { "clip_ratio/high_max": 0.00013142186152334843, "clip_ratio/high_mean": 0.00013142186152334843, "clip_ratio/low_mean": 0.0002898183399035285, "clip_ratio/low_min": 0.0002898183399035285, "clip_ratio/region_mean": 0.00042124020142687694, "completions/clipped_ratio": 0.0, "completions/max_length": 86.0, "completions/max_terminated_length": 86.0, "completions/mean_length": 54.052734375, "completions/mean_terminated_length": 54.052734375, "completions/min_length": 36.0, "completions/min_terminated_length": 36.0, "entropy": 0.1067884632696708, "epoch": 0.09708392603129445, "frac_reward_zero_std": 0.625, "grad_norm": 0.7056789398193359, "kl": 0.28939730094538796, "learning_rate": 4.753419752512072e-08, "loss": -0.0051, "num_tokens": 23259243.0, "reward": 1.22265625, "reward_std": 0.16498352587223053, "rewards/equation_reward_func/mean": 0.22265625, "rewards/equation_reward_func/std": 0.41643625497817993, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.6293188333511353, "sampling/importance_sampling_ratio/mean": 0.999923050403595, "sampling/importance_sampling_ratio/min": 0.6104380488395691, "sampling/sampling_logp_difference/max": 0.4935784339904785, "sampling/sampling_logp_difference/mean": 0.00535621726885438, "step": 364 }, { "clip_ratio/high_max": 0.0001282519531539745, "clip_ratio/high_mean": 0.0001282519531539745, "clip_ratio/low_mean": 0.0002561479793964989, "clip_ratio/low_min": 0.0002561479793964989, "clip_ratio/region_mean": 0.00038439993255047337, "completions/clipped_ratio": 0.0, "completions/max_length": 72.0, "completions/max_terminated_length": 72.0, "completions/mean_length": 53.712890625, "completions/mean_terminated_length": 53.712890625, "completions/min_length": 35.0, "completions/min_terminated_length": 35.0, "entropy": 0.10322670535080963, "epoch": 0.09761735419630156, "frac_reward_zero_std": 0.578125, "grad_norm": 0.45988044142723083, "kl": 0.2182366508576605, "learning_rate": 4.5441856801230525e-08, "loss": 0.0103, "num_tokens": 23364856.0, "reward": 1.169921875, "reward_std": 0.1906290352344513, "rewards/equation_reward_func/mean": 0.169921875, "rewards/equation_reward_func/std": 0.3759314715862274, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4551732540130615, "sampling/importance_sampling_ratio/mean": 0.9998262524604797, "sampling/importance_sampling_ratio/min": 0.4740542471408844, "sampling/sampling_logp_difference/max": 0.7464334964752197, "sampling/sampling_logp_difference/mean": 0.005158869083970785, "step": 366 }, { "clip_ratio/high_max": 0.00015984804162548648, "clip_ratio/high_mean": 0.00015984804162548648, "clip_ratio/low_mean": 9.854466124023829e-05, "clip_ratio/low_min": 9.854466124023829e-05, "clip_ratio/region_mean": 0.00025839270286572474, "completions/clipped_ratio": 0.0, "completions/max_length": 75.0, "completions/max_terminated_length": 75.0, "completions/mean_length": 53.77734375, "completions/mean_terminated_length": 53.77734375, "completions/min_length": 34.0, "completions/min_terminated_length": 34.0, "entropy": 0.11221809167828825, "epoch": 0.09815078236130868, "frac_reward_zero_std": 0.625, "grad_norm": 0.7078414559364319, "kl": 0.30065473231176537, "learning_rate": 4.3391997202891825e-08, "loss": -0.0062, "num_tokens": 23470462.0, "reward": 1.1953125, "reward_std": 0.16978386044502258, "rewards/equation_reward_func/mean": 0.1953125, "rewards/equation_reward_func/std": 0.3968288004398346, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.5995075702667236, "sampling/importance_sampling_ratio/mean": 0.9997912645339966, "sampling/importance_sampling_ratio/min": 0.47465410828590393, "sampling/sampling_logp_difference/max": 0.745168924331665, "sampling/sampling_logp_difference/mean": 0.005443467292934656, "step": 368 }, { "clip_ratio/high_max": 6.360278348438442e-05, "clip_ratio/high_mean": 6.360278348438442e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 6.360278348438442e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 74.5, "completions/max_terminated_length": 74.5, "completions/mean_length": 54.5341796875, "completions/mean_terminated_length": 54.5341796875, "completions/min_length": 37.0, "completions/min_terminated_length": 37.0, "entropy": 0.10346044186088774, "epoch": 0.09868421052631579, "frac_reward_zero_std": 0.6796875, "grad_norm": 0.4914635121822357, "kl": 0.5723805690391196, "learning_rate": 4.1385044429817966e-08, "loss": 0.0113, "num_tokens": 23682817.0, "reward": 1.12109375, "reward_std": 0.14466224610805511, "rewards/equation_reward_func/mean": 0.1220703125, "rewards/equation_reward_func/std": 0.32691892981529236, "rewards/format_reward_func/mean": 0.9990234375, "rewards/format_reward_func/std": 0.022097086533904076, "sampling/importance_sampling_ratio/max": 1.618697166442871, "sampling/importance_sampling_ratio/mean": 0.9997864067554474, "sampling/importance_sampling_ratio/min": 0.47324445843696594, "sampling/sampling_logp_difference/max": 0.748143196105957, "sampling/sampling_logp_difference/mean": 0.005407879129052162, "step": 370 }, { "clip_ratio/high_max": 0.00018945899662665196, "clip_ratio/high_mean": 0.00018945899662665196, "clip_ratio/low_mean": 0.00015853772573690448, "clip_ratio/low_min": 0.00015853772573690448, "clip_ratio/region_mean": 0.00034799672236355644, "completions/clipped_ratio": 0.0, "completions/max_length": 73.0, "completions/max_terminated_length": 73.0, "completions/mean_length": 54.654296875, "completions/mean_terminated_length": 54.654296875, "completions/min_length": 42.0, "completions/min_terminated_length": 42.0, "entropy": 0.11119612202876145, "epoch": 0.0992176386913229, "frac_reward_zero_std": 0.734375, "grad_norm": 0.6408384442329407, "kl": 0.20877801668312815, "learning_rate": 3.942141527114978e-08, "loss": -0.015, "num_tokens": 23789112.0, "reward": 1.15234375, "reward_std": 0.11921608448028564, "rewards/equation_reward_func/mean": 0.15234375, "rewards/equation_reward_func/std": 0.35970520973205566, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.443091869354248, "sampling/importance_sampling_ratio/mean": 0.9999127388000488, "sampling/importance_sampling_ratio/min": 0.5361312031745911, "sampling/sampling_logp_difference/max": 0.6233763694763184, "sampling/sampling_logp_difference/mean": 0.005300004035234451, "step": 372 }, { "clip_ratio/high_max": 6.431497038445539e-05, "clip_ratio/high_mean": 6.431497038445539e-05, "clip_ratio/low_mean": 3.135189439894425e-05, "clip_ratio/low_min": 3.135189439894425e-05, "clip_ratio/region_mean": 9.566686478339964e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 82.0, "completions/max_terminated_length": 82.0, "completions/mean_length": 54.119140625, "completions/mean_terminated_length": 54.119140625, "completions/min_length": 35.0, "completions/min_terminated_length": 35.0, "entropy": 0.1074680650813712, "epoch": 0.09975106685633002, "frac_reward_zero_std": 0.59375, "grad_norm": 0.43706050515174866, "kl": 0.20503270087970626, "learning_rate": 3.7501517518899486e-08, "loss": 0.0154, "num_tokens": 23894981.0, "reward": 1.208984375, "reward_std": 0.1902308464050293, "rewards/equation_reward_func/mean": 0.208984375, "rewards/equation_reward_func/std": 0.40698084235191345, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.455344557762146, "sampling/importance_sampling_ratio/mean": 0.999972939491272, "sampling/importance_sampling_ratio/min": 0.4449951648712158, "sampling/sampling_logp_difference/max": 0.8096919059753418, "sampling/sampling_logp_difference/mean": 0.0052422829903662205, "step": 374 }, { "clip_ratio/high_max": 0.00012764912551372417, "clip_ratio/high_mean": 0.00012764912551372417, "clip_ratio/low_mean": 6.40415172610018e-05, "clip_ratio/low_min": 6.40415172610018e-05, "clip_ratio/region_mean": 0.00019169064277472594, "completions/clipped_ratio": 0.0, "completions/max_length": 72.0, "completions/max_terminated_length": 72.0, "completions/mean_length": 54.064453125, "completions/mean_terminated_length": 54.064453125, "completions/min_length": 34.0, "completions/min_terminated_length": 34.0, "entropy": 0.10840522083971235, "epoch": 0.10028449502133713, "frac_reward_zero_std": 0.640625, "grad_norm": 0.7811239361763, "kl": 0.3093399351669682, "learning_rate": 3.562574988326342e-08, "loss": -0.0028, "num_tokens": 24000806.0, "reward": 1.142578125, "reward_std": 0.16090275347232819, "rewards/equation_reward_func/mean": 0.142578125, "rewards/equation_reward_func/std": 0.3499840497970581, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.6213926076889038, "sampling/importance_sampling_ratio/mean": 1.0001095533370972, "sampling/importance_sampling_ratio/min": 0.37080225348472595, "sampling/sampling_logp_difference/max": 0.9920864105224609, "sampling/sampling_logp_difference/mean": 0.005592584144324064, "step": 376 }, { "clip_ratio/high_max": 0.0001946542017524027, "clip_ratio/high_mean": 0.0001946542017524027, "clip_ratio/low_mean": 0.00015795392876801392, "clip_ratio/low_min": 0.00015795392876801392, "clip_ratio/region_mean": 0.00035260813052041665, "completions/clipped_ratio": 0.0, "completions/max_length": 75.0, "completions/max_terminated_length": 75.0, "completions/mean_length": 54.26171875, "completions/mean_terminated_length": 54.26171875, "completions/min_length": 35.0, "completions/min_terminated_length": 35.0, "entropy": 0.11080473537246387, "epoch": 0.10081792318634424, "frac_reward_zero_std": 0.640625, "grad_norm": 0.8036662340164185, "kl": 0.34697790671553874, "learning_rate": 3.379450190982114e-08, "loss": -0.0028, "num_tokens": 24106732.0, "reward": 1.154296875, "reward_std": 0.15801823139190674, "rewards/equation_reward_func/mean": 0.154296875, "rewards/equation_reward_func/std": 0.36158639192581177, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.636888027191162, "sampling/importance_sampling_ratio/mean": 0.9999844431877136, "sampling/importance_sampling_ratio/min": 0.4736279845237732, "sampling/sampling_logp_difference/max": 0.7473330497741699, "sampling/sampling_logp_difference/mean": 0.005764137953519821, "step": 378 }, { "clip_ratio/high_max": 0.00012605664475510517, "clip_ratio/high_mean": 0.00012605664475510517, "clip_ratio/low_mean": 0.00012994798534135852, "clip_ratio/low_min": 0.00012994798534135852, "clip_ratio/region_mean": 0.0002560046300964637, "completions/clipped_ratio": 0.0, "completions/max_length": 71.0, "completions/max_terminated_length": 71.0, "completions/mean_length": 53.802734375, "completions/mean_terminated_length": 53.802734375, "completions/min_length": 34.0, "completions/min_terminated_length": 34.0, "entropy": 0.10849014855921268, "epoch": 0.10135135135135136, "frac_reward_zero_std": 0.703125, "grad_norm": 0.5985727310180664, "kl": 0.4769698733256923, "learning_rate": 3.2008153898637255e-08, "loss": 0.0001, "num_tokens": 24212399.0, "reward": 1.138671875, "reward_std": 0.12914691865444183, "rewards/equation_reward_func/mean": 0.138671875, "rewards/equation_reward_func/std": 0.34594178199768066, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4897253513336182, "sampling/importance_sampling_ratio/mean": 1.000121831893921, "sampling/importance_sampling_ratio/min": 0.47605547308921814, "sampling/sampling_logp_difference/max": 0.7422208786010742, "sampling/sampling_logp_difference/mean": 0.005334699060767889, "step": 380 }, { "clip_ratio/high_max": 0.00013204237782499858, "clip_ratio/high_mean": 0.00013204237782499858, "clip_ratio/low_mean": 0.0002276669579765035, "clip_ratio/low_min": 0.0002276669579765035, "clip_ratio/region_mean": 0.00035970933580150205, "completions/clipped_ratio": 0.0, "completions/max_length": 74.0, "completions/max_terminated_length": 74.0, "completions/mean_length": 52.939453125, "completions/mean_terminated_length": 52.939453125, "completions/min_length": 37.0, "completions/min_terminated_length": 37.0, "entropy": 0.10587952389485306, "epoch": 0.10188477951635846, "frac_reward_zero_std": 0.609375, "grad_norm": 0.7023279070854187, "kl": 0.22818097844719887, "learning_rate": 3.026707682528365e-08, "loss": -0.0042, "num_tokens": 24317400.0, "reward": 1.20703125, "reward_std": 0.16314628720283508, "rewards/equation_reward_func/mean": 0.20703125, "rewards/equation_reward_func/std": 0.40557438135147095, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.451087236404419, "sampling/importance_sampling_ratio/mean": 1.000056266784668, "sampling/importance_sampling_ratio/min": 0.6041624546051025, "sampling/sampling_logp_difference/max": 0.503912091255188, "sampling/sampling_logp_difference/mean": 0.0053162239491939545, "step": 382 }, { "clip_ratio/high_max": 3.1928480085399417e-05, "clip_ratio/high_mean": 3.1928480085399417e-05, "clip_ratio/low_mean": 0.0001602483680471778, "clip_ratio/low_min": 0.0001602483680471778, "clip_ratio/region_mean": 0.0001921768481325772, "completions/clipped_ratio": 0.0, "completions/max_length": 78.0, "completions/max_terminated_length": 78.0, "completions/mean_length": 54.53515625, "completions/mean_terminated_length": 54.53515625, "completions/min_length": 37.0, "completions/min_terminated_length": 37.0, "entropy": 0.10885071423318651, "epoch": 0.10241820768136557, "frac_reward_zero_std": 0.625, "grad_norm": 0.9072684049606323, "kl": 0.3265605628904369, "learning_rate": 2.8571632263797745e-08, "loss": 0.001, "num_tokens": 24423626.0, "reward": 1.1640625, "reward_std": 0.1621546745300293, "rewards/equation_reward_func/mean": 0.1640625, "rewards/equation_reward_func/std": 0.37069445848464966, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4786670207977295, "sampling/importance_sampling_ratio/mean": 0.9996113777160645, "sampling/importance_sampling_ratio/min": 0.4724961221218109, "sampling/sampling_logp_difference/max": 0.7497258186340332, "sampling/sampling_logp_difference/mean": 0.005308371968567371, "step": 384 }, { "clip_ratio/high_max": 3.2679737260979084e-05, "clip_ratio/high_mean": 3.2679737260979084e-05, "clip_ratio/low_mean": 0.00019532104165086316, "clip_ratio/low_min": 0.00019532104165086316, "clip_ratio/region_mean": 0.00022800077891184224, "completions/clipped_ratio": 0.0, "completions/max_length": 79.0, "completions/max_terminated_length": 79.0, "completions/mean_length": 53.27734375, "completions/mean_terminated_length": 53.27734375, "completions/min_length": 37.0, "completions/min_terminated_length": 37.0, "entropy": 0.10969236410326427, "epoch": 0.1029516358463727, "frac_reward_zero_std": 0.6171875, "grad_norm": 0.8136389255523682, "kl": 0.2749815947479672, "learning_rate": 2.6922172311593884e-08, "loss": 0.0013, "num_tokens": 24634038.0, "reward": 1.205078125, "reward_std": 0.16476191580295563, "rewards/equation_reward_func/mean": 0.20703125, "rewards/equation_reward_func/std": 0.4051145315170288, "rewards/format_reward_func/mean": 0.998046875, "rewards/format_reward_func/std": 0.03121940791606903, "sampling/importance_sampling_ratio/max": 1.6781755685806274, "sampling/importance_sampling_ratio/mean": 0.9999711513519287, "sampling/importance_sampling_ratio/min": 0.598201721906662, "sampling/sampling_logp_difference/max": 0.5333266258239746, "sampling/sampling_logp_difference/mean": 0.005326430080458522, "step": 386 }, { "clip_ratio/high_max": 0.0001562595037588229, "clip_ratio/high_mean": 0.0001562595037588229, "clip_ratio/low_mean": 0.00016435466952518455, "clip_ratio/low_min": 0.00016435466952518455, "clip_ratio/region_mean": 0.0003206141765177664, "completions/clipped_ratio": 0.0, "completions/max_length": 79.0, "completions/max_terminated_length": 79.0, "completions/mean_length": 54.33984375, "completions/mean_terminated_length": 54.33984375, "completions/min_length": 32.0, "completions/min_terminated_length": 32.0, "entropy": 0.10665819586979018, "epoch": 0.1034850640113798, "frac_reward_zero_std": 0.828125, "grad_norm": 0.5512076616287231, "kl": 0.25673069970475304, "learning_rate": 2.5319039516341844e-08, "loss": -0.0046, "num_tokens": 24740132.0, "reward": 1.099609375, "reward_std": 0.07699654996395111, "rewards/equation_reward_func/mean": 0.099609375, "rewards/equation_reward_func/std": 0.29977133870124817, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.8591183423995972, "sampling/importance_sampling_ratio/mean": 1.000103235244751, "sampling/importance_sampling_ratio/min": 0.6897003054618835, "sampling/sampling_logp_difference/max": 0.6201024055480957, "sampling/sampling_logp_difference/mean": 0.005149331409484148, "step": 388 }, { "clip_ratio/high_max": 6.239134947665864e-05, "clip_ratio/high_mean": 6.239134947665864e-05, "clip_ratio/low_mean": 9.417718522147172e-05, "clip_ratio/low_min": 9.417718522147172e-05, "clip_ratio/region_mean": 0.00015656853469813036, "completions/clipped_ratio": 0.0, "completions/max_length": 82.0, "completions/max_terminated_length": 82.0, "completions/mean_length": 54.537109375, "completions/mean_terminated_length": 54.537109375, "completions/min_length": 38.0, "completions/min_terminated_length": 38.0, "entropy": 0.10971176334553295, "epoch": 0.10401849217638691, "frac_reward_zero_std": 0.640625, "grad_norm": 0.692693829536438, "kl": 0.2153504646072785, "learning_rate": 2.3762566804829742e-08, "loss": -0.0027, "num_tokens": 24846255.0, "reward": 1.158203125, "reward_std": 0.16714656352996826, "rewards/equation_reward_func/mean": 0.158203125, "rewards/equation_reward_func/std": 0.36528825759887695, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.7098453044891357, "sampling/importance_sampling_ratio/mean": 1.0002349615097046, "sampling/importance_sampling_ratio/min": 0.4768085777759552, "sampling/sampling_logp_difference/max": 0.7406401634216309, "sampling/sampling_logp_difference/mean": 0.0057729813270270824, "step": 390 }, { "clip_ratio/high_max": 9.298927681003179e-05, "clip_ratio/high_mean": 9.298927681003179e-05, "clip_ratio/low_mean": 9.58334163038267e-05, "clip_ratio/low_min": 9.58334163038267e-05, "clip_ratio/region_mean": 0.0001888226931138585, "completions/clipped_ratio": 0.0, "completions/max_length": 80.0, "completions/max_terminated_length": 80.0, "completions/mean_length": 53.759765625, "completions/mean_terminated_length": 53.759765625, "completions/min_length": 38.0, "completions/min_terminated_length": 38.0, "entropy": 0.11222462935580148, "epoch": 0.10455192034139403, "frac_reward_zero_std": 0.765625, "grad_norm": 0.6533622145652771, "kl": 0.30716266441676354, "learning_rate": 2.2253077413823458e-08, "loss": 0.0082, "num_tokens": 24951828.0, "reward": 1.12109375, "reward_std": 0.1058690994977951, "rewards/equation_reward_func/mean": 0.12109375, "rewards/equation_reward_func/std": 0.3265552520751953, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9998432397842407, "sampling/importance_sampling_ratio/min": 0.537998378276825, "sampling/sampling_logp_difference/max": 0.6983377933502197, "sampling/sampling_logp_difference/mean": 0.005584080703556538, "step": 392 }, { "clip_ratio/high_max": 0.00025420298010835215, "clip_ratio/high_mean": 0.00025420298010835215, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00025420298010835215, "completions/clipped_ratio": 0.0, "completions/max_length": 80.0, "completions/max_terminated_length": 80.0, "completions/mean_length": 54.27734375, "completions/mean_terminated_length": 54.27734375, "completions/min_length": 38.0, "completions/min_terminated_length": 38.0, "entropy": 0.10937931471400791, "epoch": 0.10508534850640114, "frac_reward_zero_std": 0.71875, "grad_norm": 0.5494008660316467, "kl": 0.190183505622877, "learning_rate": 2.0790884822939836e-08, "loss": -0.0054, "num_tokens": 25057778.0, "reward": 1.14453125, "reward_std": 0.1324264407157898, "rewards/equation_reward_func/mean": 0.14453125, "rewards/equation_reward_func/std": 0.35197147727012634, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.5624616146087646, "sampling/importance_sampling_ratio/mean": 1.0000663995742798, "sampling/importance_sampling_ratio/min": 0.5897343754768372, "sampling/sampling_logp_difference/max": 0.528083086013794, "sampling/sampling_logp_difference/mean": 0.005392900668084621, "step": 394 }, { "clip_ratio/high_max": 0.0002244631792159958, "clip_ratio/high_mean": 0.0002244631792159958, "clip_ratio/low_mean": 6.410540663637221e-05, "clip_ratio/low_min": 6.410540663637221e-05, "clip_ratio/region_mean": 0.00028856858585236803, "completions/clipped_ratio": 0.0, "completions/max_length": 75.0, "completions/max_terminated_length": 75.0, "completions/mean_length": 53.62109375, "completions/mean_terminated_length": 53.62109375, "completions/min_length": 35.0, "completions/min_terminated_length": 35.0, "entropy": 0.11050118981964058, "epoch": 0.10561877667140825, "frac_reward_zero_std": 0.6875, "grad_norm": 0.5563652515411377, "kl": 0.2800249651902252, "learning_rate": 1.9376292689545158e-08, "loss": 0.0042, "num_tokens": 25163296.0, "reward": 1.138671875, "reward_std": 0.13296005129814148, "rewards/equation_reward_func/mean": 0.140625, "rewards/equation_reward_func/std": 0.3479743003845215, "rewards/format_reward_func/mean": 0.998046875, "rewards/format_reward_func/std": 0.04419417306780815, "sampling/importance_sampling_ratio/max": 1.6930088996887207, "sampling/importance_sampling_ratio/mean": 1.0000758171081543, "sampling/importance_sampling_ratio/min": 0.47472190856933594, "sampling/sampling_logp_difference/max": 0.7450261116027832, "sampling/sampling_logp_difference/mean": 0.005275674629956484, "step": 396 }, { "clip_ratio/high_max": 3.299023632684516e-05, "clip_ratio/high_mean": 3.299023632684516e-05, "clip_ratio/low_mean": 6.681774134954644e-05, "clip_ratio/low_min": 6.681774134954644e-05, "clip_ratio/region_mean": 9.98079776763916e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 83.0, "completions/max_terminated_length": 83.0, "completions/mean_length": 53.294921875, "completions/mean_terminated_length": 53.294921875, "completions/min_length": 36.0, "completions/min_terminated_length": 36.0, "entropy": 0.10927490248448318, "epoch": 0.10615220483641537, "frac_reward_zero_std": 0.71875, "grad_norm": 0.6097186207771301, "kl": 0.21666139115889868, "learning_rate": 1.800959478569422e-08, "loss": 0.0028, "num_tokens": 25268447.0, "reward": 1.166015625, "reward_std": 0.12276645004749298, "rewards/equation_reward_func/mean": 0.166015625, "rewards/equation_reward_func/std": 0.3724585771560669, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.5868266820907593, "sampling/importance_sampling_ratio/mean": 0.9999920129776001, "sampling/importance_sampling_ratio/min": 0.41749459505081177, "sampling/sampling_logp_difference/max": 0.8734836578369141, "sampling/sampling_logp_difference/mean": 0.005354772787541151, "step": 398 }, { "clip_ratio/high_max": 3.36292723659426e-05, "clip_ratio/high_mean": 3.36292723659426e-05, "clip_ratio/low_mean": 9.74817505468511e-05, "clip_ratio/low_min": 9.74817505468511e-05, "clip_ratio/region_mean": 0.00013111102291279368, "completions/clipped_ratio": 0.0, "completions/max_length": 84.0, "completions/max_terminated_length": 84.0, "completions/mean_length": 54.10546875, "completions/mean_terminated_length": 54.10546875, "completions/min_length": 41.0, "completions/min_terminated_length": 41.0, "entropy": 0.11135919060972002, "epoch": 0.10668563300142248, "frac_reward_zero_std": 0.734375, "grad_norm": 0.5380083918571472, "kl": 0.3635715709792243, "learning_rate": 1.6691074937121407e-08, "loss": -0.0035, "num_tokens": 25374229.0, "reward": 1.12109375, "reward_std": 0.11152878403663635, "rewards/equation_reward_func/mean": 0.123046875, "rewards/equation_reward_func/std": 0.32881227135658264, "rewards/format_reward_func/mean": 0.998046875, "rewards/format_reward_func/std": 0.04419417306780815, "sampling/importance_sampling_ratio/max": 1.587953805923462, "sampling/importance_sampling_ratio/mean": 1.000246524810791, "sampling/importance_sampling_ratio/min": 0.6871500015258789, "sampling/sampling_logp_difference/max": 0.4624462127685547, "sampling/sampling_logp_difference/mean": 0.004950111731886864, "step": 400 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 3.0127742017308872e-05, "clip_ratio/low_min": 3.0127742017308872e-05, "clip_ratio/region_mean": 3.0127742017308872e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 73.5, "completions/max_terminated_length": 73.5, "completions/mean_length": 54.755859375, "completions/mean_terminated_length": 54.755859375, "completions/min_length": 38.5, "completions/min_terminated_length": 38.5, "entropy": 0.10915559923483266, "epoch": 0.10721906116642958, "frac_reward_zero_std": 0.6796875, "grad_norm": 0.5684436559677124, "kl": 0.20094259290231598, "learning_rate": 1.5421006964298377e-08, "loss": -0.0052, "num_tokens": 25586915.0, "reward": 1.142578125, "reward_std": 0.1426895670592785, "rewards/equation_reward_func/mean": 0.142578125, "rewards/equation_reward_func/std": 0.34958136081695557, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4456849694252014, "sampling/importance_sampling_ratio/mean": 1.000089704990387, "sampling/importance_sampling_ratio/min": 0.506356343626976, "sampling/sampling_logp_difference/max": 0.6825828552246094, "sampling/sampling_logp_difference/mean": 0.0053960594814270735, "step": 402 }, { "clip_ratio/high_max": 0.00018773043929185305, "clip_ratio/high_mean": 0.00018773043929185305, "clip_ratio/low_mean": 3.163756142991284e-05, "clip_ratio/low_min": 3.163756142991284e-05, "clip_ratio/region_mean": 0.00021936800072176589, "completions/clipped_ratio": 0.0, "completions/max_length": 78.0, "completions/max_terminated_length": 78.0, "completions/mean_length": 53.34375, "completions/mean_terminated_length": 53.34375, "completions/min_length": 36.0, "completions/min_terminated_length": 36.0, "entropy": 0.10881674703624514, "epoch": 0.1077524893314367, "frac_reward_zero_std": 0.75, "grad_norm": 0.5433355569839478, "kl": 0.23106293090515667, "learning_rate": 1.4199654625568575e-08, "loss": 0.0069, "num_tokens": 25692203.0, "reward": 1.126953125, "reward_std": 0.1148107573390007, "rewards/equation_reward_func/mean": 0.126953125, "rewards/equation_reward_func/std": 0.33324605226516724, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.6753840446472168, "sampling/importance_sampling_ratio/mean": 1.0001516342163086, "sampling/importance_sampling_ratio/min": 0.47411516308784485, "sampling/sampling_logp_difference/max": 0.746304988861084, "sampling/sampling_logp_difference/mean": 0.005414518993347883, "step": 404 }, { "clip_ratio/high_max": 6.571563426405191e-05, "clip_ratio/high_mean": 6.571563426405191e-05, "clip_ratio/low_mean": 3.233734363069137e-05, "clip_ratio/low_min": 3.233734363069137e-05, "clip_ratio/region_mean": 9.805297789474328e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 71.0, "completions/max_terminated_length": 71.0, "completions/mean_length": 54.330078125, "completions/mean_terminated_length": 54.330078125, "completions/min_length": 34.0, "completions/min_terminated_length": 34.0, "entropy": 0.10553992167115211, "epoch": 0.10828591749644381, "frac_reward_zero_std": 0.578125, "grad_norm": 0.7051840424537659, "kl": 0.25712460403641063, "learning_rate": 1.302727156237224e-08, "loss": -0.0031, "num_tokens": 25798260.0, "reward": 1.212890625, "reward_std": 0.18944953382015228, "rewards/equation_reward_func/mean": 0.21484375, "rewards/equation_reward_func/std": 0.4111155867576599, "rewards/format_reward_func/mean": 0.998046875, "rewards/format_reward_func/std": 0.04419417306780815, "sampling/importance_sampling_ratio/max": 1.6173588037490845, "sampling/importance_sampling_ratio/mean": 0.9998912215232849, "sampling/importance_sampling_ratio/min": 0.5355166792869568, "sampling/sampling_logp_difference/max": 0.6245231628417969, "sampling/sampling_logp_difference/mean": 0.005070769228041172, "step": 406 }, { "clip_ratio/high_max": 0.0001589058018806908, "clip_ratio/high_mean": 0.0001589058018806908, "clip_ratio/low_mean": 6.55909728569289e-05, "clip_ratio/low_min": 6.55909728569289e-05, "clip_ratio/region_mean": 0.00022449677473761968, "completions/clipped_ratio": 0.0, "completions/max_length": 76.0, "completions/max_terminated_length": 76.0, "completions/mean_length": 54.080078125, "completions/mean_terminated_length": 54.080078125, "completions/min_length": 36.0, "completions/min_terminated_length": 36.0, "entropy": 0.10741599255965815, "epoch": 0.10881934566145092, "frac_reward_zero_std": 0.75, "grad_norm": 0.5971543788909912, "kl": 0.3184280472083224, "learning_rate": 1.1904101246571874e-08, "loss": 0.0085, "num_tokens": 25904117.0, "reward": 1.099609375, "reward_std": 0.1145443245768547, "rewards/equation_reward_func/mean": 0.099609375, "rewards/equation_reward_func/std": 0.29977133870124817, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.46078360080719, "sampling/importance_sampling_ratio/mean": 1.0001788139343262, "sampling/importance_sampling_ratio/min": 0.41946613788604736, "sampling/sampling_logp_difference/max": 0.8687725067138672, "sampling/sampling_logp_difference/mean": 0.004916239529848099, "step": 408 }, { "clip_ratio/high_max": 0.00029215401931044954, "clip_ratio/high_mean": 0.00029215401931044954, "clip_ratio/low_mean": 6.396731219461394e-05, "clip_ratio/low_min": 6.396731219461394e-05, "clip_ratio/region_mean": 0.0003561213315050635, "completions/clipped_ratio": 0.0, "completions/max_length": 72.0, "completions/max_terminated_length": 72.0, "completions/mean_length": 53.44140625, "completions/mean_terminated_length": 53.44140625, "completions/min_length": 37.0, "completions/min_terminated_length": 37.0, "entropy": 0.106752199638221, "epoch": 0.10935277382645804, "frac_reward_zero_std": 0.5625, "grad_norm": 0.6034308075904846, "kl": 0.19704193456305397, "learning_rate": 1.0830376929889612e-08, "loss": -0.002, "num_tokens": 26009479.0, "reward": 1.2109375, "reward_std": 0.19444583356380463, "rewards/equation_reward_func/mean": 0.2109375, "rewards/equation_reward_func/std": 0.4083731174468994, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4476516246795654, "sampling/importance_sampling_ratio/mean": 0.9999210834503174, "sampling/importance_sampling_ratio/min": 0.5376265048980713, "sampling/sampling_logp_difference/max": 0.6205911636352539, "sampling/sampling_logp_difference/mean": 0.005288798827677965, "step": 410 }, { "clip_ratio/high_max": 0.00013152075957299935, "clip_ratio/high_mean": 0.00013152075957299935, "clip_ratio/low_mean": 0.00012655667119866444, "clip_ratio/low_min": 0.00012655667119866444, "clip_ratio/region_mean": 0.0002580774307716638, "completions/clipped_ratio": 0.0, "completions/max_length": 79.0, "completions/max_terminated_length": 79.0, "completions/mean_length": 53.400390625, "completions/mean_terminated_length": 53.400390625, "completions/min_length": 33.0, "completions/min_terminated_length": 33.0, "entropy": 0.11002827166683143, "epoch": 0.10988620199146515, "frac_reward_zero_std": 0.671875, "grad_norm": 0.6455569267272949, "kl": 0.20942417076892322, "learning_rate": 9.806321595467598e-09, "loss": -0.0013, "num_tokens": 26114732.0, "reward": 1.24609375, "reward_std": 0.1438756138086319, "rewards/equation_reward_func/mean": 0.24609375, "rewards/equation_reward_func/std": 0.4311550557613373, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.5866832733154297, "sampling/importance_sampling_ratio/mean": 1.0001810789108276, "sampling/importance_sampling_ratio/min": 0.6237314343452454, "sampling/sampling_logp_difference/max": 0.47203534841537476, "sampling/sampling_logp_difference/mean": 0.005659942049533129, "step": 412 }, { "clip_ratio/high_max": 0.00012916017052096626, "clip_ratio/high_mean": 0.00012916017052096626, "clip_ratio/low_mean": 9.72550284738342e-05, "clip_ratio/low_min": 9.72550284738342e-05, "clip_ratio/region_mean": 0.00022641519899480045, "completions/clipped_ratio": 0.0, "completions/max_length": 76.0, "completions/max_terminated_length": 76.0, "completions/mean_length": 54.03515625, "completions/mean_terminated_length": 54.03515625, "completions/min_length": 36.0, "completions/min_terminated_length": 36.0, "entropy": 0.11343364355464776, "epoch": 0.11041963015647226, "frac_reward_zero_std": 0.71875, "grad_norm": 0.781673789024353, "kl": 0.21153342889414894, "learning_rate": 8.832147911560173e-09, "loss": 0.0025, "num_tokens": 26220518.0, "reward": 1.181640625, "reward_std": 0.12671181559562683, "rewards/equation_reward_func/mean": 0.181640625, "rewards/equation_reward_func/std": 0.38592514395713806, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.8482966423034668, "sampling/importance_sampling_ratio/mean": 0.9999876022338867, "sampling/importance_sampling_ratio/min": 0.3686046600341797, "sampling/sampling_logp_difference/max": 0.9980306625366211, "sampling/sampling_logp_difference/mean": 0.006071026436984539, "step": 414 }, { "clip_ratio/high_max": 6.601988570764661e-05, "clip_ratio/high_mean": 6.601988570764661e-05, "clip_ratio/low_mean": 0.00015977946976717148, "clip_ratio/low_min": 0.00015977946976717148, "clip_ratio/region_mean": 0.0002257993554748181, "completions/clipped_ratio": 0.0, "completions/max_length": 81.0, "completions/max_terminated_length": 81.0, "completions/mean_length": 54.458984375, "completions/mean_terminated_length": 54.458984375, "completions/min_length": 35.0, "completions/min_terminated_length": 35.0, "entropy": 0.11500270126594438, "epoch": 0.11095305832147938, "frac_reward_zero_std": 0.625, "grad_norm": 0.5759050846099854, "kl": 0.18094956999023756, "learning_rate": 7.908058187368726e-09, "loss": -0.0042, "num_tokens": 26326585.0, "reward": 1.16796875, "reward_std": 0.1624211072921753, "rewards/equation_reward_func/mean": 0.16796875, "rewards/equation_reward_func/std": 0.374204158782959, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9998392462730408, "sampling/importance_sampling_ratio/min": 0.4201279282569885, "sampling/sampling_logp_difference/max": 0.8671960830688477, "sampling/sampling_logp_difference/mean": 0.005369527265429497, "step": 416 }, { "clip_ratio/high_max": 6.738985151362915e-05, "clip_ratio/high_mean": 6.738985151362915e-05, "clip_ratio/low_mean": 3.200204810127616e-05, "clip_ratio/low_min": 3.200204810127616e-05, "clip_ratio/region_mean": 9.939189961490531e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 74.5, "completions/max_terminated_length": 74.5, "completions/mean_length": 53.6337890625, "completions/mean_terminated_length": 53.6337890625, "completions/min_length": 36.0, "completions/min_terminated_length": 36.0, "entropy": 0.11205452390842968, "epoch": 0.11148648648648649, "frac_reward_zero_std": 0.6484375, "grad_norm": 0.6088562607765198, "kl": 0.2512584278980891, "learning_rate": 7.0342443310273665e-09, "loss": -0.003, "num_tokens": 26537490.0, "reward": 1.166015625, "reward_std": 0.1546947881579399, "rewards/equation_reward_func/mean": 0.166015625, "rewards/equation_reward_func/std": 0.37186430394649506, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.714827835559845, "sampling/importance_sampling_ratio/mean": 0.9997610151767731, "sampling/importance_sampling_ratio/min": 0.3313564211130142, "sampling/sampling_logp_difference/max": 1.1132968664169312, "sampling/sampling_logp_difference/mean": 0.005667505552992225, "step": 418 }, { "clip_ratio/high_max": 0.0001591860103265693, "clip_ratio/high_mean": 0.0001591860103265693, "clip_ratio/low_mean": 0.00012875603473124406, "clip_ratio/low_min": 0.00012875603473124406, "clip_ratio/region_mean": 0.00028794204505781334, "completions/clipped_ratio": 0.0, "completions/max_length": 79.0, "completions/max_terminated_length": 79.0, "completions/mean_length": 54.89453125, "completions/mean_terminated_length": 54.89453125, "completions/min_length": 38.0, "completions/min_terminated_length": 38.0, "entropy": 0.11335018173687988, "epoch": 0.1120199146514936, "frac_reward_zero_std": 0.65625, "grad_norm": 0.7120157480239868, "kl": 0.21040037108792198, "learning_rate": 6.210887809749099e-09, "loss": 0.0046, "num_tokens": 26643868.0, "reward": 1.154296875, "reward_std": 0.1618293821811676, "rewards/equation_reward_func/mean": 0.15625, "rewards/equation_reward_func/std": 0.36344730854034424, "rewards/format_reward_func/mean": 0.998046875, "rewards/format_reward_func/std": 0.04419417306780815, "sampling/importance_sampling_ratio/max": 1.6395080089569092, "sampling/importance_sampling_ratio/mean": 0.9999812841415405, "sampling/importance_sampling_ratio/min": 0.609165370464325, "sampling/sampling_logp_difference/max": 0.4956655502319336, "sampling/sampling_logp_difference/mean": 0.00603404501453042, "step": 420 }, { "clip_ratio/high_max": 3.229974148174127e-05, "clip_ratio/high_mean": 3.229974148174127e-05, "clip_ratio/low_mean": 0.00028592673091528315, "clip_ratio/low_min": 0.00028592673091528315, "clip_ratio/region_mean": 0.00031822647239702445, "completions/clipped_ratio": 0.0, "completions/max_length": 84.0, "completions/max_terminated_length": 84.0, "completions/mean_length": 53.38671875, "completions/mean_terminated_length": 53.38671875, "completions/min_length": 40.0, "completions/min_terminated_length": 40.0, "entropy": 0.11383476294577122, "epoch": 0.11255334281650071, "frac_reward_zero_std": 0.640625, "grad_norm": 0.6631842851638794, "kl": 0.37177798876331913, "learning_rate": 5.4381596121399476e-09, "loss": -0.0003, "num_tokens": 26749146.0, "reward": 1.18359375, "reward_std": 0.16149252653121948, "rewards/equation_reward_func/mean": 0.18359375, "rewards/equation_reward_func/std": 0.3875311613082886, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.5404492616653442, "sampling/importance_sampling_ratio/mean": 1.000014305114746, "sampling/importance_sampling_ratio/min": 0.5083003640174866, "sampling/sampling_logp_difference/max": 0.676682710647583, "sampling/sampling_logp_difference/mean": 0.005789338611066341, "step": 422 }, { "clip_ratio/high_max": 0.00022921719937585294, "clip_ratio/high_mean": 0.00022921719937585294, "clip_ratio/low_mean": 3.142282690128518e-05, "clip_ratio/low_min": 3.142282690128518e-05, "clip_ratio/region_mean": 0.00026064002627713815, "completions/clipped_ratio": 0.0, "completions/max_length": 80.0, "completions/max_terminated_length": 80.0, "completions/mean_length": 53.37109375, "completions/mean_terminated_length": 53.37109375, "completions/min_length": 37.0, "completions/min_terminated_length": 37.0, "entropy": 0.11201548162433836, "epoch": 0.11308677098150782, "frac_reward_zero_std": 0.671875, "grad_norm": 0.7947490811347961, "kl": 0.1786341749959522, "learning_rate": 4.716220212689332e-09, "loss": 0.0087, "num_tokens": 26854416.0, "reward": 1.2109375, "reward_std": 0.14013366401195526, "rewards/equation_reward_func/mean": 0.212890625, "rewards/equation_reward_func/std": 0.409751296043396, "rewards/format_reward_func/mean": 0.998046875, "rewards/format_reward_func/std": 0.04419417306780815, "sampling/importance_sampling_ratio/max": 1.7253990173339844, "sampling/importance_sampling_ratio/mean": 1.0001569986343384, "sampling/importance_sampling_ratio/min": 0.4751533269882202, "sampling/sampling_logp_difference/max": 0.7441177368164062, "sampling/sampling_logp_difference/mean": 0.004982165526598692, "step": 424 }, { "clip_ratio/high_max": 0.0001316617094239013, "clip_ratio/high_mean": 0.0001316617094239013, "clip_ratio/low_mean": 6.346045928593312e-05, "clip_ratio/low_min": 6.346045928593312e-05, "clip_ratio/region_mean": 0.00019512216870983443, "completions/clipped_ratio": 0.0, "completions/max_length": 74.0, "completions/max_terminated_length": 74.0, "completions/mean_length": 54.45703125, "completions/mean_terminated_length": 54.45703125, "completions/min_length": 36.0, "completions/min_terminated_length": 36.0, "entropy": 0.10795113237367736, "epoch": 0.11362019914651493, "frac_reward_zero_std": 0.75, "grad_norm": 0.6196188926696777, "kl": 0.259122790561782, "learning_rate": 4.045219538443778e-09, "loss": -0.0036, "num_tokens": 26960570.0, "reward": 1.201171875, "reward_std": 0.10888905078172684, "rewards/equation_reward_func/mean": 0.203125, "rewards/equation_reward_func/std": 0.4027182459831238, "rewards/format_reward_func/mean": 0.998046875, "rewards/format_reward_func/std": 0.04419417306780815, "sampling/importance_sampling_ratio/max": 1.415486216545105, "sampling/importance_sampling_ratio/mean": 0.9997314214706421, "sampling/importance_sampling_ratio/min": 0.5374058485031128, "sampling/sampling_logp_difference/max": 0.6210017204284668, "sampling/sampling_logp_difference/mean": 0.005382740870118141, "step": 426 }, { "clip_ratio/high_max": 0.00012776548586164912, "clip_ratio/high_mean": 0.00012776548586164912, "clip_ratio/low_mean": 9.751586993742321e-05, "clip_ratio/low_min": 9.751586993742321e-05, "clip_ratio/region_mean": 0.0002252813557990723, "completions/clipped_ratio": 0.0, "completions/max_length": 70.0, "completions/max_terminated_length": 70.0, "completions/mean_length": 54.076171875, "completions/mean_terminated_length": 54.076171875, "completions/min_length": 37.0, "completions/min_terminated_length": 37.0, "entropy": 0.10863094880349106, "epoch": 0.11415362731152205, "frac_reward_zero_std": 0.75, "grad_norm": 0.5494850277900696, "kl": 0.19497828578783405, "learning_rate": 3.4252969378714134e-09, "loss": 0.0076, "num_tokens": 27066361.0, "reward": 1.162109375, "reward_std": 0.12052415311336517, "rewards/equation_reward_func/mean": 0.162109375, "rewards/equation_reward_func/std": 0.3689115643501282, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.5807867050170898, "sampling/importance_sampling_ratio/mean": 1.0001335144042969, "sampling/importance_sampling_ratio/min": 0.47290024161338806, "sampling/sampling_logp_difference/max": 0.748870849609375, "sampling/sampling_logp_difference/mean": 0.005263046361505985, "step": 428 }, { "clip_ratio/high_max": 3.237503309113284e-05, "clip_ratio/high_mean": 3.237503309113284e-05, "clip_ratio/low_mean": 0.0001267518253169126, "clip_ratio/low_min": 0.0001267518253169126, "clip_ratio/region_mean": 0.00015912685840804543, "completions/clipped_ratio": 0.0, "completions/max_length": 73.0, "completions/max_terminated_length": 73.0, "completions/mean_length": 53.65234375, "completions/mean_terminated_length": 53.65234375, "completions/min_length": 37.0, "completions/min_terminated_length": 37.0, "entropy": 0.10955962331758605, "epoch": 0.11468705547652916, "frac_reward_zero_std": 0.640625, "grad_norm": 0.6626308560371399, "kl": 0.3944896838317315, "learning_rate": 2.856581151922943e-09, "loss": -0.0035, "num_tokens": 27171863.0, "reward": 1.1484375, "reward_std": 0.15551269054412842, "rewards/equation_reward_func/mean": 0.1484375, "rewards/equation_reward_func/std": 0.35588082671165466, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.397895097732544, "sampling/importance_sampling_ratio/mean": 1.000016212463379, "sampling/importance_sampling_ratio/min": 0.32679349184036255, "sampling/sampling_logp_difference/max": 1.11842679977417, "sampling/sampling_logp_difference/mean": 0.005313401110470295, "step": 430 }, { "clip_ratio/high_max": 9.902074493260847e-05, "clip_ratio/high_mean": 9.902074493260847e-05, "clip_ratio/low_mean": 9.541076724417508e-05, "clip_ratio/low_min": 9.541076724417508e-05, "clip_ratio/region_mean": 0.00019443151217678355, "completions/clipped_ratio": 0.0, "completions/max_length": 78.0, "completions/max_terminated_length": 78.0, "completions/mean_length": 53.7265625, "completions/mean_terminated_length": 53.7265625, "completions/min_length": 34.0, "completions/min_terminated_length": 34.0, "entropy": 0.11242794824971093, "epoch": 0.11522048364153627, "frac_reward_zero_std": 0.734375, "grad_norm": 0.646879255771637, "kl": 0.20790828329821429, "learning_rate": 2.339190287295678e-09, "loss": -0.0056, "num_tokens": 27277387.0, "reward": 1.1484375, "reward_std": 0.12263103574514389, "rewards/equation_reward_func/mean": 0.1484375, "rewards/equation_reward_func/std": 0.35588082671165466, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.63825523853302, "sampling/importance_sampling_ratio/mean": 0.9999201893806458, "sampling/importance_sampling_ratio/min": 0.5359142422676086, "sampling/sampling_logp_difference/max": 0.6237812042236328, "sampling/sampling_logp_difference/mean": 0.005299155134707689, "step": 432 }, { "clip_ratio/high_max": 6.528276329239209e-05, "clip_ratio/high_mean": 6.528276329239209e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 6.528276329239209e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 75.0, "completions/max_terminated_length": 75.0, "completions/mean_length": 53.53515625, "completions/mean_terminated_length": 53.53515625, "completions/min_length": 38.0, "completions/min_terminated_length": 38.0, "entropy": 0.1100023119813866, "epoch": 0.11575391180654339, "frac_reward_zero_std": 0.7265625, "grad_norm": 0.3693109452724457, "kl": 0.21212409436702728, "learning_rate": 1.8732317919060715e-09, "loss": -0.0003, "num_tokens": 27488303.0, "reward": 1.1318359375, "reward_std": 0.11800135672092438, "rewards/equation_reward_func/mean": 0.1318359375, "rewards/equation_reward_func/std": 0.3379475623369217, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.7036836743354797, "sampling/importance_sampling_ratio/mean": 0.9997938275337219, "sampling/importance_sampling_ratio/min": 0.5718834400177002, "sampling/sampling_logp_difference/max": 0.5988638401031494, "sampling/sampling_logp_difference/mean": 0.005505367880687118, "step": 434 }, { "clip_ratio/high_max": 9.71159930082245e-05, "clip_ratio/high_mean": 9.71159930082245e-05, "clip_ratio/low_mean": 9.421712214437623e-05, "clip_ratio/low_min": 9.421712214437623e-05, "clip_ratio/region_mean": 0.00019133311515260074, "completions/clipped_ratio": 0.0, "completions/max_length": 77.0, "completions/max_terminated_length": 77.0, "completions/mean_length": 55.1484375, "completions/mean_terminated_length": 55.1484375, "completions/min_length": 34.0, "completions/min_terminated_length": 34.0, "entropy": 0.10632556552688281, "epoch": 0.1162873399715505, "frac_reward_zero_std": 0.640625, "grad_norm": 0.608108401298523, "kl": 0.18701807078388002, "learning_rate": 1.4588024325756788e-09, "loss": 0.0035, "num_tokens": 27594915.0, "reward": 1.138671875, "reward_std": 0.1637316346168518, "rewards/equation_reward_func/mean": 0.138671875, "rewards/equation_reward_func/std": 0.34594178199768066, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4311141967773438, "sampling/importance_sampling_ratio/mean": 0.9998852014541626, "sampling/importance_sampling_ratio/min": 0.5086729526519775, "sampling/sampling_logp_difference/max": 0.6759500503540039, "sampling/sampling_logp_difference/mean": 0.0051868068985641, "step": 436 }, { "clip_ratio/high_max": 9.734939602721069e-05, "clip_ratio/high_mean": 9.734939602721069e-05, "clip_ratio/low_mean": 6.28134552648084e-05, "clip_ratio/low_min": 6.28134552648084e-05, "clip_ratio/region_mean": 0.0001601628512920191, "completions/clipped_ratio": 0.0, "completions/max_length": 74.0, "completions/max_terminated_length": 74.0, "completions/mean_length": 54.189453125, "completions/mean_terminated_length": 54.189453125, "completions/min_length": 35.0, "completions/min_terminated_length": 35.0, "entropy": 0.11053359135985374, "epoch": 0.1168207681365576, "frac_reward_zero_std": 0.671875, "grad_norm": 0.6285626292228699, "kl": 0.24602288214696777, "learning_rate": 1.0959882749354277e-09, "loss": 0.0011, "num_tokens": 27700876.0, "reward": 1.185546875, "reward_std": 0.143285870552063, "rewards/equation_reward_func/mean": 0.185546875, "rewards/equation_reward_func/std": 0.38912075757980347, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.620995283126831, "sampling/importance_sampling_ratio/mean": 0.9998245239257812, "sampling/importance_sampling_ratio/min": 0.44262897968292236, "sampling/sampling_logp_difference/max": 0.8150234222412109, "sampling/sampling_logp_difference/mean": 0.005426466464996338, "step": 438 }, { "clip_ratio/high_max": 0.00012761998287816014, "clip_ratio/high_mean": 0.00012761998287816014, "clip_ratio/low_mean": 0.000159376420520453, "clip_ratio/low_min": 0.000159376420520453, "clip_ratio/region_mean": 0.00028699640339861315, "completions/clipped_ratio": 0.0, "completions/max_length": 90.0, "completions/max_terminated_length": 90.0, "completions/mean_length": 53.83984375, "completions/mean_terminated_length": 53.83984375, "completions/min_length": 42.0, "completions/min_terminated_length": 42.0, "entropy": 0.10714658308360311, "epoch": 0.11735419630156473, "frac_reward_zero_std": 0.703125, "grad_norm": 0.45735955238342285, "kl": 0.2908731723825137, "learning_rate": 7.848646655519986e-10, "loss": -0.005, "num_tokens": 27806522.0, "reward": 1.203125, "reward_std": 0.1334155797958374, "rewards/equation_reward_func/mean": 0.205078125, "rewards/equation_reward_func/std": 0.4041535556316376, "rewards/format_reward_func/mean": 0.998046875, "rewards/format_reward_func/std": 0.04419417306780815, "sampling/importance_sampling_ratio/max": 1.4523879289627075, "sampling/importance_sampling_ratio/mean": 0.9998242259025574, "sampling/importance_sampling_ratio/min": 0.6908398270606995, "sampling/sampling_logp_difference/max": 0.37320899963378906, "sampling/sampling_logp_difference/mean": 0.004832516424357891, "step": 440 }, { "clip_ratio/high_max": 6.629579002037644e-05, "clip_ratio/high_mean": 6.629579002037644e-05, "clip_ratio/low_mean": 0.00016045565522896746, "clip_ratio/low_min": 0.00016045565522896746, "clip_ratio/region_mean": 0.0002267514452493439, "completions/clipped_ratio": 0.0, "completions/max_length": 83.0, "completions/max_terminated_length": 83.0, "completions/mean_length": 54.224609375, "completions/mean_terminated_length": 54.224609375, "completions/min_length": 36.0, "completions/min_terminated_length": 36.0, "entropy": 0.10644441656768322, "epoch": 0.11788762446657183, "frac_reward_zero_std": 0.671875, "grad_norm": 0.6941789984703064, "kl": 0.20567356712288326, "learning_rate": 5.254962162804799e-10, "loss": 0.0061, "num_tokens": 27912469.0, "reward": 1.169921875, "reward_std": 0.14243090152740479, "rewards/equation_reward_func/mean": 0.173828125, "rewards/equation_reward_func/std": 0.3793322443962097, "rewards/format_reward_func/mean": 0.99609375, "rewards/format_reward_func/std": 0.06243881583213806, "sampling/importance_sampling_ratio/max": 1.7774187326431274, "sampling/importance_sampling_ratio/mean": 1.0001775026321411, "sampling/importance_sampling_ratio/min": 0.47465693950653076, "sampling/sampling_logp_difference/max": 0.7451629638671875, "sampling/sampling_logp_difference/mean": 0.005035408306866884, "step": 442 }, { "clip_ratio/high_max": 9.787768552390237e-05, "clip_ratio/high_mean": 9.787768552390237e-05, "clip_ratio/low_mean": 9.694968078595896e-05, "clip_ratio/low_min": 9.694968078595896e-05, "clip_ratio/region_mean": 0.00019482736630986133, "completions/clipped_ratio": 0.0, "completions/max_length": 76.0, "completions/max_terminated_length": 76.0, "completions/mean_length": 54.001953125, "completions/mean_terminated_length": 54.001953125, "completions/min_length": 36.0, "completions/min_terminated_length": 36.0, "entropy": 0.11240475873152415, "epoch": 0.11842105263157894, "frac_reward_zero_std": 0.6875, "grad_norm": 0.5599156618118286, "kl": 0.2706406813942724, "learning_rate": 3.1793679084632375e-10, "loss": -0.0021, "num_tokens": 28018262.0, "reward": 1.150390625, "reward_std": 0.14459514617919922, "rewards/equation_reward_func/mean": 0.150390625, "rewards/equation_reward_func/std": 0.35780346393585205, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.6746656894683838, "sampling/importance_sampling_ratio/mean": 1.0000989437103271, "sampling/importance_sampling_ratio/min": 0.32599928975105286, "sampling/sampling_logp_difference/max": 1.1208600997924805, "sampling/sampling_logp_difference/mean": 0.005868655163794756, "step": 444 }, { "clip_ratio/high_max": 0.00012872741596462825, "clip_ratio/high_mean": 0.00012872741596462825, "clip_ratio/low_mean": 3.079576223778228e-05, "clip_ratio/low_min": 3.079576223778228e-05, "clip_ratio/region_mean": 0.00015952317820241055, "completions/clipped_ratio": 0.0, "completions/max_length": 76.0, "completions/max_terminated_length": 76.0, "completions/mean_length": 53.478515625, "completions/mean_terminated_length": 53.478515625, "completions/min_length": 36.0, "completions/min_terminated_length": 36.0, "entropy": 0.11034609129031499, "epoch": 0.11895448079658606, "frac_reward_zero_std": 0.671875, "grad_norm": 0.5812860131263733, "kl": 0.24613611793352497, "learning_rate": 1.6222949365926608e-10, "loss": -0.0054, "num_tokens": 28123555.0, "reward": 1.16796875, "reward_std": 0.14814552664756775, "rewards/equation_reward_func/mean": 0.16796875, "rewards/equation_reward_func/std": 0.374204158782959, "rewards/format_reward_func/mean": 1.0, "rewards/format_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4527826309204102, "sampling/importance_sampling_ratio/mean": 0.9999824166297913, "sampling/importance_sampling_ratio/min": 0.47327739000320435, "sampling/sampling_logp_difference/max": 0.7480735778808594, "sampling/sampling_logp_difference/mean": 0.0052461326122283936, "step": 446 }, { "clip_ratio/high_max": 0.00015984166788661646, "clip_ratio/high_mean": 0.00015984166788661646, "clip_ratio/low_mean": 0.0001612306417276462, "clip_ratio/low_min": 0.0001612306417276462, "clip_ratio/region_mean": 0.00032107230961426266, "completions/clipped_ratio": 0.0, "completions/max_length": 79.0, "completions/max_terminated_length": 79.0, "completions/mean_length": 54.671875, "completions/mean_terminated_length": 54.671875, "completions/min_length": 36.0, "completions/min_terminated_length": 36.0, "entropy": 0.1114695995218224, "epoch": 0.11948790896159317, "frac_reward_zero_std": 0.6875, "grad_norm": 0.6841051578521729, "kl": 0.2143436846219831, "learning_rate": 5.84066608615985e-11, "loss": 0.0066, "num_tokens": 28229851.0, "reward": 1.14453125, "reward_std": 0.14432993531227112, "rewards/equation_reward_func/mean": 0.146484375, "rewards/equation_reward_func/std": 0.35393697023391724, "rewards/format_reward_func/mean": 0.998046875, "rewards/format_reward_func/std": 0.04419417306780815, "sampling/importance_sampling_ratio/max": 1.3628417253494263, "sampling/importance_sampling_ratio/mean": 0.9998343586921692, "sampling/importance_sampling_ratio/min": 0.5373474359512329, "sampling/sampling_logp_difference/max": 0.6211104393005371, "sampling/sampling_logp_difference/mean": 0.005433104000985622, "step": 448 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 2.8965357665179505e-05, "clip_ratio/low_min": 2.8965357665179505e-05, "clip_ratio/region_mean": 2.8965357665179505e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 85.5, "completions/max_terminated_length": 85.5, "completions/mean_length": 54.6015625, "completions/mean_terminated_length": 54.6015625, "completions/min_length": 41.0, "completions/min_terminated_length": 41.0, "entropy": 0.11345378682017326, "epoch": 0.12002133712660028, "frac_reward_zero_std": 0.6640625, "grad_norm": 0.5929685235023499, "kl": 0.30157056140402955, "learning_rate": 6.489853613067531e-12, "loss": -0.0012, "num_tokens": 28442211.0, "reward": 1.162109375, "reward_std": 0.14864009618759155, "rewards/equation_reward_func/mean": 0.1630859375, "rewards/equation_reward_func/std": 0.3668737858533859, "rewards/format_reward_func/mean": 0.9990234375, "rewards/format_reward_func/std": 0.022097086533904076, "sampling/importance_sampling_ratio/max": 1.5475803017616272, "sampling/importance_sampling_ratio/mean": 1.0000160336494446, "sampling/importance_sampling_ratio/min": 0.6126877665519714, "sampling/sampling_logp_difference/max": 0.557793140411377, "sampling/sampling_logp_difference/mean": 0.0051607247442007065, "step": 450 }, { "epoch": 0.12002133712660028, "step": 450, "total_flos": 0.0, "train_loss": 0.010528181331465021, "train_runtime": 3465.3708, "train_samples_per_second": 37.399, "train_steps_per_second": 0.13 } ], "logging_steps": 2, "max_steps": 450, "num_input_tokens_seen": 28442211, "num_train_epochs": 1, "save_steps": 25, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": true, "should_training_stop": true }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 16, "trial_name": null, "trial_params": null }