{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 2.0, "eval_steps": 500, "global_step": 300, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.525, "completions/max_length": 512.0, "completions/max_terminated_length": 378.9, "completions/mean_length": 457.9125, "completions/mean_terminated_length": 322.60096435546876, "completions/min_length": 351.5, "completions/min_terminated_length": 249.1, "entropy": 0.3948360087350011, "epoch": 0.06666666666666667, "frac_reward_zero_std": 0.05, "grad_norm": 2.03125, "kl": 0.04399079610357148, "learning_rate": 9.7e-06, "loss": -0.004937342554330826, "num_tokens": 49697.0, "reward": 0.65526123046875, "reward_std": 0.4537597060203552, "rewards/JointRewardFunction/mean": 0.65526123046875, "rewards/JointRewardFunction/std": 0.45375972241163254, "step": 10, "step_time": 22.810993660400346 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.5, "completions/max_length": 508.7, "completions/max_terminated_length": 417.6, "completions/mean_length": 443.675, "completions/mean_terminated_length": 342.31012573242185, "completions/min_length": 326.9, "completions/min_terminated_length": 275.7, "entropy": 0.31964702922850846, "epoch": 0.13333333333333333, "frac_reward_zero_std": 0.25, "grad_norm": 0.1982421875, "kl": 0.09551988991443068, "learning_rate": 9.366666666666668e-06, "loss": 0.00889739990234375, "num_tokens": 97803.0, "reward": 0.7364013671875, "reward_std": 0.4511200308799744, "rewards/JointRewardFunction/mean": 0.7364013671875, "rewards/JointRewardFunction/std": 0.4511200487613678, "step": 20, "step_time": 23.522369655099464 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.675, "completions/max_length": 512.0, "completions/max_terminated_length": 379.6, "completions/mean_length": 477.4875, "completions/mean_terminated_length": 350.68857421875, "completions/min_length": 371.0, "completions/min_terminated_length": 319.8, "entropy": 0.40151252076029775, "epoch": 0.2, "frac_reward_zero_std": 0.35, "grad_norm": 0.1630859375, "kl": 0.11964645721018315, "learning_rate": 9.033333333333334e-06, "loss": -0.0028077010065317154, "num_tokens": 148394.0, "reward": 0.9680908203125, "reward_std": 0.40886188447475436, "rewards/JointRewardFunction/mean": 0.9680908203125, "rewards/JointRewardFunction/std": 0.40886189937591555, "step": 30, "step_time": 23.04281468010122 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.4, "completions/max_length": 512.0, "completions/max_terminated_length": 469.3, "completions/mean_length": 438.0875, "completions/mean_terminated_length": 393.87953491210936, "completions/min_length": 335.8, "completions/min_terminated_length": 335.8, "entropy": 0.35483993515372275, "epoch": 0.26666666666666666, "frac_reward_zero_std": 0.25, "grad_norm": 2.34375, "kl": 0.0885015036445111, "learning_rate": 8.700000000000001e-06, "loss": 0.03651041984558105, "num_tokens": 197537.0, "reward": 0.9726318359375, "reward_std": 0.38605129569768903, "rewards/JointRewardFunction/mean": 0.9726318359375, "rewards/JointRewardFunction/std": 0.3860512986779213, "step": 40, "step_time": 23.371826623300876 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.075, "completions/max_length": 416.7, "completions/max_terminated_length": 387.6, "completions/mean_length": 307.5, "completions/mean_terminated_length": 296.2386932373047, "completions/min_length": 221.0, "completions/min_terminated_length": 221.0, "entropy": 0.3532023083418608, "epoch": 0.3333333333333333, "frac_reward_zero_std": 0.2, "grad_norm": 2.484375, "kl": 0.1458138041663915, "learning_rate": 8.366666666666667e-06, "loss": 0.058735775947570804, "num_tokens": 235441.0, "reward": 0.911224365234375, "reward_std": 0.44634594917297366, "rewards/JointRewardFunction/mean": 0.911224365234375, "rewards/JointRewardFunction/std": 0.4463459521532059, "step": 50, "step_time": 19.590083589001733 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 439.0, "completions/max_terminated_length": 411.3, "completions/mean_length": 314.675, "completions/mean_terminated_length": 302.16786193847656, "completions/min_length": 213.5, "completions/min_terminated_length": 213.5, "entropy": 0.33970522740855813, "epoch": 0.4, "frac_reward_zero_std": 0.25, "grad_norm": 2.21875, "kl": 0.13768328949809075, "learning_rate": 8.033333333333335e-06, "loss": 0.024235518276691438, "num_tokens": 273659.0, "reward": 1.012158203125, "reward_std": 0.34636138631030916, "rewards/JointRewardFunction/mean": 1.012158203125, "rewards/JointRewardFunction/std": 0.3463613982312381, "step": 60, "step_time": 20.71416906509876 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0375, "completions/max_length": 405.4, "completions/max_terminated_length": 397.6, "completions/mean_length": 306.85, "completions/mean_terminated_length": 298.9619079589844, "completions/min_length": 215.4, "completions/min_terminated_length": 215.4, "entropy": 0.36216176860034466, "epoch": 0.4666666666666667, "frac_reward_zero_std": 0.5, "grad_norm": 2.375, "kl": 0.11585550024174154, "learning_rate": 7.7e-06, "loss": -0.0011418772861361503, "num_tokens": 311055.0, "reward": 1.108551025390625, "reward_std": 0.31149580857600084, "rewards/JointRewardFunction/mean": 1.108551025390625, "rewards/JointRewardFunction/std": 0.311495810066117, "step": 70, "step_time": 19.105042513498848 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0375, "completions/max_length": 425.2, "completions/max_terminated_length": 399.6, "completions/mean_length": 303.65, "completions/mean_terminated_length": 296.1714324951172, "completions/min_length": 221.7, "completions/min_terminated_length": 221.7, "entropy": 0.458731491304934, "epoch": 0.5333333333333333, "frac_reward_zero_std": 0.25, "grad_norm": 1.8515625, "kl": 0.12144780233502388, "learning_rate": 7.3666666666666676e-06, "loss": 0.04609571397304535, "num_tokens": 347759.0, "reward": 1.0232421875, "reward_std": 0.39059739662334325, "rewards/JointRewardFunction/mean": 1.0232421875, "rewards/JointRewardFunction/std": 0.39059740854427216, "step": 80, "step_time": 20.152326823800102 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.025, "completions/max_length": 458.2, "completions/max_terminated_length": 434.3, "completions/mean_length": 325.7, "completions/mean_terminated_length": 320.2678619384766, "completions/min_length": 231.3, "completions/min_terminated_length": 231.3, "entropy": 0.414226458594203, "epoch": 0.6, "frac_reward_zero_std": 0.45, "grad_norm": 2.71875, "kl": 0.11777536268346012, "learning_rate": 7.033333333333334e-06, "loss": 0.017658862471580505, "num_tokens": 386695.0, "reward": 1.0622802734375, "reward_std": 0.37295620441436766, "rewards/JointRewardFunction/mean": 1.0622802734375, "rewards/JointRewardFunction/std": 0.3729562103748322, "step": 90, "step_time": 21.401791930101172 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0375, "completions/max_length": 419.9, "completions/max_terminated_length": 405.6, "completions/mean_length": 305.0625, "completions/mean_terminated_length": 297.75535888671874, "completions/min_length": 201.6, "completions/min_terminated_length": 201.6, "entropy": 0.4645195137709379, "epoch": 0.6666666666666666, "frac_reward_zero_std": 0.45, "grad_norm": 3.21875, "kl": 0.11028626803308725, "learning_rate": 6.700000000000001e-06, "loss": 0.019249849021434784, "num_tokens": 423284.0, "reward": 1.095147705078125, "reward_std": 0.3025559388101101, "rewards/JointRewardFunction/mean": 1.095147705078125, "rewards/JointRewardFunction/std": 0.302555950731039, "step": 100, "step_time": 19.82219038610201 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0375, "completions/max_length": 437.4, "completions/max_terminated_length": 425.9, "completions/mean_length": 325.925, "completions/mean_terminated_length": 319.71428833007815, "completions/min_length": 227.9, "completions/min_terminated_length": 227.9, "entropy": 0.43129814630374314, "epoch": 0.7333333333333333, "frac_reward_zero_std": 0.5, "grad_norm": 1.40625, "kl": 0.1031634843442589, "learning_rate": 6.366666666666668e-06, "loss": 0.026633352041244507, "num_tokens": 462986.0, "reward": 1.043157958984375, "reward_std": 0.33559322796645574, "rewards/JointRewardFunction/mean": 1.043157958984375, "rewards/JointRewardFunction/std": 0.3355932369071525, "step": 110, "step_time": 20.435491250798805 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.05, "completions/max_length": 444.4, "completions/max_terminated_length": 440.1, "completions/mean_length": 337.825, "completions/mean_terminated_length": 330.73964538574216, "completions/min_length": 233.7, "completions/min_terminated_length": 233.7, "entropy": 0.43917944487184285, "epoch": 0.8, "frac_reward_zero_std": 0.45, "grad_norm": 4.03125, "kl": 0.1373384237056598, "learning_rate": 6.033333333333335e-06, "loss": 0.051127517223358156, "num_tokens": 504032.0, "reward": 1.03193359375, "reward_std": 0.326281476020813, "rewards/JointRewardFunction/mean": 1.03193359375, "rewards/JointRewardFunction/std": 0.3262814819812775, "step": 120, "step_time": 20.735252708198097 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.075, "completions/max_length": 412.0, "completions/max_terminated_length": 392.5, "completions/mean_length": 312.775, "completions/mean_terminated_length": 300.4857147216797, "completions/min_length": 222.2, "completions/min_terminated_length": 222.2, "entropy": 0.4507128844037652, "epoch": 0.8666666666666667, "frac_reward_zero_std": 0.5, "grad_norm": 2.09375, "kl": 0.11878000139258801, "learning_rate": 5.7e-06, "loss": 0.011486038565635681, "num_tokens": 544434.0, "reward": 1.053509521484375, "reward_std": 0.31438804864883424, "rewards/JointRewardFunction/mean": 1.053509521484375, "rewards/JointRewardFunction/std": 0.31438804864883424, "step": 130, "step_time": 19.64918669259896 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.05, "completions/max_length": 439.1, "completions/max_terminated_length": 435.6, "completions/mean_length": 333.05, "completions/mean_terminated_length": 327.24464416503906, "completions/min_length": 234.3, "completions/min_terminated_length": 234.3, "entropy": 0.40997486785054205, "epoch": 0.9333333333333333, "frac_reward_zero_std": 0.4, "grad_norm": 2.84375, "kl": 0.10414444564376027, "learning_rate": 5.366666666666666e-06, "loss": 0.011959856748580933, "num_tokens": 584866.0, "reward": 1.083343505859375, "reward_std": 0.3227477639913559, "rewards/JointRewardFunction/mean": 1.083343505859375, "rewards/JointRewardFunction/std": 0.3227477639913559, "step": 140, "step_time": 20.672387204799815 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0875, "completions/max_length": 434.4, "completions/max_terminated_length": 412.6, "completions/mean_length": 322.45, "completions/mean_terminated_length": 308.4941680908203, "completions/min_length": 227.0, "completions/min_terminated_length": 227.0, "entropy": 0.4216183707118034, "epoch": 1.0, "frac_reward_zero_std": 0.35, "grad_norm": 3.234375, "kl": 0.10422139009460807, "learning_rate": 5.033333333333333e-06, "loss": 0.009627737849950791, "num_tokens": 623390.0, "reward": 1.072760009765625, "reward_std": 0.298717375023989, "rewards/JointRewardFunction/mean": 1.072760009765625, "rewards/JointRewardFunction/std": 0.29871737800422127, "step": 150, "step_time": 20.43400644650028 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.075, "completions/max_length": 477.3, "completions/max_terminated_length": 441.4, "completions/mean_length": 332.3375, "completions/mean_terminated_length": 314.32142944335936, "completions/min_length": 212.2, "completions/min_terminated_length": 212.2, "entropy": 0.4680627126246691, "epoch": 1.0666666666666667, "frac_reward_zero_std": 0.45, "grad_norm": 0.054443359375, "kl": 0.09179632470477372, "learning_rate": 4.7e-06, "loss": 0.01482328325510025, "num_tokens": 663349.0, "reward": 1.0511962890625, "reward_std": 0.33187133751343934, "rewards/JointRewardFunction/mean": 1.0511962890625, "rewards/JointRewardFunction/std": 0.33187134347390385, "step": 160, "step_time": 22.216746885597967 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.05, "completions/max_length": 431.7, "completions/max_terminated_length": 406.5, "completions/mean_length": 307.6625, "completions/mean_terminated_length": 298.221435546875, "completions/min_length": 207.6, "completions/min_terminated_length": 207.6, "entropy": 0.4726201189681888, "epoch": 1.1333333333333333, "frac_reward_zero_std": 0.6, "grad_norm": 2.078125, "kl": 0.10375995303038507, "learning_rate": 4.366666666666667e-06, "loss": 0.008281752467155457, "num_tokens": 699830.0, "reward": 1.084967041015625, "reward_std": 0.2650434322655201, "rewards/JointRewardFunction/mean": 1.084967041015625, "rewards/JointRewardFunction/std": 0.26504343524575236, "step": 170, "step_time": 20.110693615402123 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 379.8, "completions/max_terminated_length": 379.8, "completions/mean_length": 297.7875, "completions/mean_terminated_length": 297.7875, "completions/min_length": 209.8, "completions/min_terminated_length": 209.8, "entropy": 0.4800571627914906, "epoch": 1.2, "frac_reward_zero_std": 0.55, "grad_norm": 2.859375, "kl": 0.10157871665433049, "learning_rate": 4.033333333333333e-06, "loss": 0.015658122301101685, "num_tokens": 735313.0, "reward": 1.125604248046875, "reward_std": 0.25403361991047857, "rewards/JointRewardFunction/mean": 1.125604248046875, "rewards/JointRewardFunction/std": 0.2540336221456528, "step": 180, "step_time": 18.15690856939873 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 442.3, "completions/max_terminated_length": 417.9, "completions/mean_length": 322.4875, "completions/mean_terminated_length": 312.0000061035156, "completions/min_length": 221.9, "completions/min_terminated_length": 221.9, "entropy": 0.4642798786982894, "epoch": 1.2666666666666666, "frac_reward_zero_std": 0.55, "grad_norm": 2.1875, "kl": 0.0930135941482149, "learning_rate": 3.7e-06, "loss": 0.022286814451217652, "num_tokens": 773840.0, "reward": 1.0874267578125, "reward_std": 0.2804231091577094, "rewards/JointRewardFunction/mean": 1.0874267578125, "rewards/JointRewardFunction/std": 0.28042311511817386, "step": 190, "step_time": 20.867724073099453 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.075, "completions/max_length": 448.1, "completions/max_terminated_length": 431.5, "completions/mean_length": 345.2375, "completions/mean_terminated_length": 335.31964416503905, "completions/min_length": 235.1, "completions/min_terminated_length": 235.1, "entropy": 0.4752088913694024, "epoch": 1.3333333333333333, "frac_reward_zero_std": 0.5, "grad_norm": 2.140625, "kl": 0.08606207200791686, "learning_rate": 3.366666666666667e-06, "loss": 0.019437333941459654, "num_tokens": 814411.0, "reward": 1.0824462890625, "reward_std": 0.3011944264173508, "rewards/JointRewardFunction/mean": 1.0824462890625, "rewards/JointRewardFunction/std": 0.3011944264173508, "step": 200, "step_time": 21.037474609902347 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1125, "completions/max_length": 447.2, "completions/max_terminated_length": 440.3, "completions/mean_length": 341.525, "completions/mean_terminated_length": 324.12596130371094, "completions/min_length": 227.0, "completions/min_terminated_length": 227.0, "entropy": 0.45753496792167425, "epoch": 1.4, "frac_reward_zero_std": 0.45, "grad_norm": 1.7265625, "kl": 0.08462402573786676, "learning_rate": 3.0333333333333337e-06, "loss": 0.03857302367687225, "num_tokens": 854993.0, "reward": 1.0694580078125, "reward_std": 0.32212619576603174, "rewards/JointRewardFunction/mean": 1.0694580078125, "rewards/JointRewardFunction/std": 0.32212621364742516, "step": 210, "step_time": 20.958584549299484 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0875, "completions/max_length": 439.8, "completions/max_terminated_length": 409.0, "completions/mean_length": 325.825, "completions/mean_terminated_length": 312.76429138183596, "completions/min_length": 230.1, "completions/min_terminated_length": 230.1, "entropy": 0.43106496389955284, "epoch": 1.4666666666666668, "frac_reward_zero_std": 0.7, "grad_norm": 1.84375, "kl": 0.08686103990767151, "learning_rate": 2.7000000000000004e-06, "loss": 0.019956868886947633, "num_tokens": 894887.0, "reward": 1.037451171875, "reward_std": 0.27579919978743417, "rewards/JointRewardFunction/mean": 1.037451171875, "rewards/JointRewardFunction/std": 0.2757992057478987, "step": 220, "step_time": 20.81888391800021 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 370.2, "completions/max_terminated_length": 357.7, "completions/mean_length": 287.3625, "completions/mean_terminated_length": 284.64464416503904, "completions/min_length": 212.9, "completions/min_terminated_length": 212.9, "entropy": 0.5203873571008444, "epoch": 1.5333333333333332, "frac_reward_zero_std": 0.7, "grad_norm": 2.4375, "kl": 0.09830583925358952, "learning_rate": 2.3666666666666667e-06, "loss": 0.013263462483882904, "num_tokens": 929688.0, "reward": 1.1615234375, "reward_std": 0.1927501901984215, "rewards/JointRewardFunction/mean": 1.1615234375, "rewards/JointRewardFunction/std": 0.19275019615888594, "step": 230, "step_time": 17.954591237399292 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1375, "completions/max_length": 455.5, "completions/max_terminated_length": 442.9, "completions/mean_length": 355.65, "completions/mean_terminated_length": 341.68095703125, "completions/min_length": 251.1, "completions/min_terminated_length": 251.1, "entropy": 0.42119636284187434, "epoch": 1.6, "frac_reward_zero_std": 0.35, "grad_norm": 1.78125, "kl": 0.08470410050358623, "learning_rate": 2.0333333333333335e-06, "loss": 0.01756674200296402, "num_tokens": 972840.0, "reward": 0.989080810546875, "reward_std": 0.37197236269712447, "rewards/JointRewardFunction/mean": 0.989080810546875, "rewards/JointRewardFunction/std": 0.37197236269712447, "step": 240, "step_time": 21.474622961001295 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0875, "completions/max_length": 462.3, "completions/max_terminated_length": 428.0, "completions/mean_length": 342.35, "completions/mean_terminated_length": 324.6696472167969, "completions/min_length": 234.8, "completions/min_terminated_length": 234.8, "entropy": 0.4915314789861441, "epoch": 1.6666666666666665, "frac_reward_zero_std": 0.3, "grad_norm": 1.2109375, "kl": 0.08292658934369683, "learning_rate": 1.7000000000000002e-06, "loss": 0.021336786448955536, "num_tokens": 1013548.0, "reward": 1.00546875, "reward_std": 0.3482314422726631, "rewards/JointRewardFunction/mean": 1.00546875, "rewards/JointRewardFunction/std": 0.3482314541935921, "step": 250, "step_time": 21.594003204298495 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1125, "completions/max_length": 458.6, "completions/max_terminated_length": 413.7, "completions/mean_length": 339.1875, "completions/mean_terminated_length": 320.0369110107422, "completions/min_length": 236.0, "completions/min_terminated_length": 236.0, "entropy": 0.4891188982874155, "epoch": 1.7333333333333334, "frac_reward_zero_std": 0.45, "grad_norm": 1.515625, "kl": 0.08876022514887154, "learning_rate": 1.3666666666666668e-06, "loss": 0.04018712341785431, "num_tokens": 1053691.0, "reward": 1.0763916015625, "reward_std": 0.33348723202943803, "rewards/JointRewardFunction/mean": 1.0763916015625, "rewards/JointRewardFunction/std": 0.33348723202943803, "step": 260, "step_time": 21.527129449401038 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.05, "completions/max_length": 424.1, "completions/max_terminated_length": 398.5, "completions/mean_length": 326.7125, "completions/mean_terminated_length": 318.79107971191405, "completions/min_length": 235.0, "completions/min_terminated_length": 235.0, "entropy": 0.5027944987639785, "epoch": 1.8, "frac_reward_zero_std": 0.4, "grad_norm": 1.4140625, "kl": 0.09775767312385142, "learning_rate": 1.0333333333333333e-06, "loss": 0.022160670161247252, "num_tokens": 1093636.0, "reward": 1.073553466796875, "reward_std": 0.2811176300048828, "rewards/JointRewardFunction/mean": 1.073553466796875, "rewards/JointRewardFunction/std": 0.2811176344752312, "step": 270, "step_time": 19.999046050798643 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.05, "completions/max_length": 418.1, "completions/max_terminated_length": 399.8, "completions/mean_length": 323.325, "completions/mean_terminated_length": 314.7875, "completions/min_length": 233.0, "completions/min_terminated_length": 233.0, "entropy": 0.46924052368849517, "epoch": 1.8666666666666667, "frac_reward_zero_std": 0.55, "grad_norm": 1.890625, "kl": 0.09717398346401751, "learning_rate": 7.000000000000001e-07, "loss": 0.011738638579845428, "num_tokens": 1132610.0, "reward": 1.1536865234375, "reward_std": 0.23539954721927642, "rewards/JointRewardFunction/mean": 1.1536865234375, "rewards/JointRewardFunction/std": 0.2353995516896248, "step": 280, "step_time": 20.17876625719837 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1375, "completions/max_length": 465.2, "completions/max_terminated_length": 430.9, "completions/mean_length": 340.55, "completions/mean_terminated_length": 316.59702758789064, "completions/min_length": 220.1, "completions/min_terminated_length": 220.1, "entropy": 0.48896827585995195, "epoch": 1.9333333333333333, "frac_reward_zero_std": 0.5, "grad_norm": 0.08544921875, "kl": 0.09501179384533316, "learning_rate": 3.666666666666667e-07, "loss": 0.011296895891427993, "num_tokens": 1175610.0, "reward": 1.069940185546875, "reward_std": 0.29926753640174864, "rewards/JointRewardFunction/mean": 1.069940185546875, "rewards/JointRewardFunction/std": 0.2992675393819809, "step": 290, "step_time": 21.805314568298854 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.075, "completions/max_length": 427.1, "completions/max_terminated_length": 417.8, "completions/mean_length": 321.225, "completions/mean_terminated_length": 312.3041687011719, "completions/min_length": 233.6, "completions/min_terminated_length": 233.6, "entropy": 0.49822530038654805, "epoch": 2.0, "frac_reward_zero_std": 0.5, "grad_norm": 2.28125, "kl": 0.09350393395870923, "learning_rate": 3.333333333333334e-08, "loss": 0.0006398491561412812, "num_tokens": 1214508.0, "reward": 1.014300537109375, "reward_std": 0.3215805938467383, "rewards/JointRewardFunction/mean": 1.014300537109375, "rewards/JointRewardFunction/std": 0.3215805938467383, "step": 300, "step_time": 20.291091689400854 } ], "logging_steps": 10, "max_steps": 300, "num_input_tokens_seen": 1214508, "num_train_epochs": 2, "save_steps": 10, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": true, "should_training_stop": true }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 1, "trial_name": null, "trial_params": null }