{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 2.0, "eval_steps": 500, "global_step": 300, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.675, "completions/max_length": 512.0, "completions/max_terminated_length": 446.8, "completions/mean_length": 486.375, "completions/mean_terminated_length": 398.4283386230469, "completions/min_length": 402.8, "completions/min_terminated_length": 351.6, "entropy": 0.2849034793674946, "epoch": 0.06666666666666667, "frac_reward_zero_std": 0.1, "grad_norm": 1.4609375, "kl": 0.031243230529071297, "learning_rate": 9.7e-06, "loss": -0.00440123975276947, "num_tokens": 51778.0, "reward": 0.8364257484674453, "reward_std": 0.48435992896556856, "rewards/JointRewardFunction/mean": 0.8364257484674453, "rewards/JointRewardFunction/std": 0.48435992896556856, "step": 10, "step_time": 23.126090779408695 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.8125, "completions/max_length": 512.0, "completions/max_terminated_length": 324.8, "completions/mean_length": 497.1375, "completions/mean_terminated_length": 311.3, "completions/min_length": 450.0, "completions/min_terminated_length": 296.4, "entropy": 0.22146789729595184, "epoch": 0.13333333333333333, "frac_reward_zero_std": 0.25, "grad_norm": 0.0830078125, "kl": 0.04523785075871274, "learning_rate": 9.366666666666668e-06, "loss": -0.002875373512506485, "num_tokens": 105309.0, "reward": 0.8499999582767487, "reward_std": 0.4174339771270752, "rewards/JointRewardFunction/mean": 0.8499999582767487, "rewards/JointRewardFunction/std": 0.41743398904800416, "step": 20, "step_time": 23.475473206402967 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.675, "completions/max_length": 512.0, "completions/max_terminated_length": 431.8, "completions/mean_length": 484.2125, "completions/mean_terminated_length": 398.3142883300781, "completions/min_length": 408.1, "completions/min_terminated_length": 356.9, "entropy": 0.3164555594325066, "epoch": 0.2, "frac_reward_zero_std": 0.3, "grad_norm": 1.3828125, "kl": 0.047980465402361006, "learning_rate": 9.033333333333334e-06, "loss": 0.015557542443275452, "num_tokens": 155462.0, "reward": 0.9499999523162842, "reward_std": 0.45912776291370394, "rewards/JointRewardFunction/mean": 0.9499999523162842, "rewards/JointRewardFunction/std": 0.45912776291370394, "step": 30, "step_time": 23.220224900517497 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.725, "completions/max_length": 511.0, "completions/max_terminated_length": 371.1, "completions/mean_length": 492.0625, "completions/mean_terminated_length": 351.725, "completions/min_length": 430.7, "completions/min_terminated_length": 328.3, "entropy": 0.25967293605208397, "epoch": 0.26666666666666666, "frac_reward_zero_std": 0.55, "grad_norm": 0.0771484375, "kl": 0.04111148213269189, "learning_rate": 8.700000000000001e-06, "loss": 0.000460333563387394, "num_tokens": 208139.0, "reward": 1.1499999523162843, "reward_std": 0.2668025135993958, "rewards/JointRewardFunction/mean": 1.1499999523162843, "rewards/JointRewardFunction/std": 0.26680250763893126, "step": 40, "step_time": 23.507141841895645 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.525, "completions/max_length": 512.0, "completions/max_terminated_length": 470.6, "completions/mean_length": 452.7125, "completions/mean_terminated_length": 406.9440521240234, "completions/min_length": 346.0, "completions/min_terminated_length": 346.0, "entropy": 0.2724090193398297, "epoch": 0.3333333333333333, "frac_reward_zero_std": 0.35, "grad_norm": 1.8984375, "kl": 0.05550626695621759, "learning_rate": 8.366666666666667e-06, "loss": 0.027076172828674316, "num_tokens": 257072.0, "reward": 1.0874999523162843, "reward_std": 0.4246312826871872, "rewards/JointRewardFunction/mean": 1.0874999523162843, "rewards/JointRewardFunction/std": 0.42463127076625823, "step": 50, "step_time": 23.53056020727963 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.45, "completions/max_length": 502.9, "completions/max_terminated_length": 450.5, "completions/mean_length": 445.05, "completions/mean_terminated_length": 394.2227416992188, "completions/min_length": 326.8, "completions/min_terminated_length": 326.8, "entropy": 0.24588112123310565, "epoch": 0.4, "frac_reward_zero_std": 0.7, "grad_norm": 0.06494140625, "kl": 0.06986886935774236, "learning_rate": 8.033333333333335e-06, "loss": 0.012386786192655564, "num_tokens": 306076.0, "reward": 1.199999952316284, "reward_std": 0.19487359523773193, "rewards/JointRewardFunction/mean": 1.199999952316284, "rewards/JointRewardFunction/std": 0.19487358927726744, "step": 60, "step_time": 23.163046325504546 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.4125, "completions/max_length": 507.0, "completions/max_terminated_length": 398.0, "completions/mean_length": 439.875, "completions/mean_terminated_length": 353.8702453613281, "completions/min_length": 354.4, "completions/min_terminated_length": 303.2, "entropy": 0.24171723760664462, "epoch": 0.4666666666666667, "frac_reward_zero_std": 0.75, "grad_norm": 0.05224609375, "kl": 0.055925692187156525, "learning_rate": 7.7e-06, "loss": -0.0038671787828207017, "num_tokens": 355062.0, "reward": 1.1874999523162841, "reward_std": 0.152357017993927, "rewards/JointRewardFunction/mean": 1.1874999523162841, "rewards/JointRewardFunction/std": 0.1523570090532303, "step": 70, "step_time": 23.317241401202047 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.3625, "completions/max_length": 512.0, "completions/max_terminated_length": 471.8, "completions/mean_length": 446.3875, "completions/mean_terminated_length": 415.53476867675784, "completions/min_length": 356.8, "completions/min_terminated_length": 356.8, "entropy": 0.28650506315752866, "epoch": 0.5333333333333333, "frac_reward_zero_std": 0.8, "grad_norm": 1.78125, "kl": 0.05017680691089481, "learning_rate": 7.3666666666666676e-06, "loss": 0.00010831728577613831, "num_tokens": 404445.0, "reward": 1.2249999523162842, "reward_std": 0.1334012567996979, "rewards/JointRewardFunction/mean": 1.2249999523162842, "rewards/JointRewardFunction/std": 0.13340125381946563, "step": 80, "step_time": 23.661325349108665 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.3625, "completions/max_length": 500.2, "completions/max_terminated_length": 476.5, "completions/mean_length": 440.225, "completions/mean_terminated_length": 414.4039306640625, "completions/min_length": 351.2, "completions/min_terminated_length": 351.2, "entropy": 0.2955667129717767, "epoch": 0.6, "frac_reward_zero_std": 0.7, "grad_norm": 1.3125, "kl": 0.048677809379296375, "learning_rate": 7.033333333333334e-06, "loss": -0.002417333424091339, "num_tokens": 453811.0, "reward": 1.0874999523162843, "reward_std": 0.25500799119472506, "rewards/JointRewardFunction/mean": 1.0874999523162843, "rewards/JointRewardFunction/std": 0.2550079971551895, "step": 90, "step_time": 23.215478483188782 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.425, "completions/max_length": 512.0, "completions/max_terminated_length": 464.2, "completions/mean_length": 445.8875, "completions/mean_terminated_length": 404.58286437988284, "completions/min_length": 355.6, "completions/min_terminated_length": 355.6, "entropy": 0.3539797434583306, "epoch": 0.6666666666666666, "frac_reward_zero_std": 0.65, "grad_norm": 1.546875, "kl": 0.04463185045169667, "learning_rate": 6.700000000000001e-06, "loss": -0.0038278792053461075, "num_tokens": 504366.0, "reward": 1.1624999523162842, "reward_std": 0.2765199363231659, "rewards/JointRewardFunction/mean": 1.1624999523162842, "rewards/JointRewardFunction/std": 0.27651992738246917, "step": 100, "step_time": 23.610984305897727 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.55, "completions/max_length": 512.0, "completions/max_terminated_length": 482.5, "completions/mean_length": 470.625, "completions/mean_terminated_length": 440.21571960449216, "completions/min_length": 395.6, "completions/min_terminated_length": 395.6, "entropy": 0.35651795882731674, "epoch": 0.7333333333333333, "frac_reward_zero_std": 0.7, "grad_norm": 1.2109375, "kl": 0.06428715067449957, "learning_rate": 6.366666666666668e-06, "loss": 0.0074311748147010805, "num_tokens": 554964.0, "reward": 1.1624999523162842, "reward_std": 0.2765199363231659, "rewards/JointRewardFunction/mean": 1.1624999523162842, "rewards/JointRewardFunction/std": 0.27651992738246917, "step": 110, "step_time": 23.372107661084737 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.4375, "completions/max_length": 512.0, "completions/max_terminated_length": 470.2, "completions/mean_length": 449.0625, "completions/mean_terminated_length": 413.7138153076172, "completions/min_length": 356.1, "completions/min_terminated_length": 356.1, "entropy": 0.43109525088220835, "epoch": 0.8, "frac_reward_zero_std": 0.7, "grad_norm": 0.052001953125, "kl": 0.056606468232348564, "learning_rate": 6.033333333333335e-06, "loss": 0.0022208547219634057, "num_tokens": 603153.0, "reward": 1.137499952316284, "reward_std": 0.2684998393058777, "rewards/JointRewardFunction/mean": 1.137499952316284, "rewards/JointRewardFunction/std": 0.2684998393058777, "step": 120, "step_time": 23.521636679096265 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.625, "completions/max_length": 512.0, "completions/max_terminated_length": 377.7, "completions/mean_length": 477.2875, "completions/mean_terminated_length": 346.51047973632814, "completions/min_length": 413.0, "completions/min_terminated_length": 310.6, "entropy": 0.4086992660537362, "epoch": 0.8666666666666667, "frac_reward_zero_std": 0.5, "grad_norm": 1.5546875, "kl": 0.0436876044026576, "learning_rate": 5.7e-06, "loss": 0.0012062092311680317, "num_tokens": 655588.0, "reward": 1.0499999523162842, "reward_std": 0.3375131905078888, "rewards/JointRewardFunction/mean": 1.0499999523162842, "rewards/JointRewardFunction/std": 0.33751317858695984, "step": 130, "step_time": 23.583847255594446 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.6625, "completions/max_length": 512.0, "completions/max_terminated_length": 418.1, "completions/mean_length": 481.65, "completions/mean_terminated_length": 388.0383331298828, "completions/min_length": 393.5, "completions/min_terminated_length": 342.3, "entropy": 0.4292863380163908, "epoch": 0.9333333333333333, "frac_reward_zero_std": 0.55, "grad_norm": 0.033203125, "kl": 0.04462884225649759, "learning_rate": 5.366666666666666e-06, "loss": 0.0003389778081327677, "num_tokens": 706752.0, "reward": 1.1499999523162843, "reward_std": 0.2966939389705658, "rewards/JointRewardFunction/mean": 1.1499999523162843, "rewards/JointRewardFunction/std": 0.2966939300298691, "step": 140, "step_time": 23.60956783029542 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.6375, "completions/max_length": 512.0, "completions/max_terminated_length": 423.6, "completions/mean_length": 480.75, "completions/mean_terminated_length": 391.1654815673828, "completions/min_length": 409.6, "completions/min_terminated_length": 358.4, "entropy": 0.4061064410954714, "epoch": 1.0, "frac_reward_zero_std": 0.65, "grad_norm": 2.0625, "kl": 0.04093303249683231, "learning_rate": 5.033333333333333e-06, "loss": 0.0008978299796581268, "num_tokens": 757524.0, "reward": 1.1874999523162841, "reward_std": 0.24493902325630187, "rewards/JointRewardFunction/mean": 1.1874999523162841, "rewards/JointRewardFunction/std": 0.24493901431560516, "step": 150, "step_time": 23.596201564991496 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.5875, "completions/max_length": 512.0, "completions/max_terminated_length": 360.4, "completions/mean_length": 470.7, "completions/mean_terminated_length": 326.9873840332031, "completions/min_length": 390.9, "completions/min_terminated_length": 288.5, "entropy": 0.4068562785163522, "epoch": 1.0666666666666667, "frac_reward_zero_std": 0.65, "grad_norm": 0.041748046875, "kl": 0.05161676681600511, "learning_rate": 4.7e-06, "loss": 0.005649929866194725, "num_tokens": 807096.0, "reward": 1.1624999523162842, "reward_std": 0.25586684942245486, "rewards/JointRewardFunction/mean": 1.1624999523162842, "rewards/JointRewardFunction/std": 0.2558668404817581, "step": 160, "step_time": 23.44238963140524 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.6375, "completions/max_length": 512.0, "completions/max_terminated_length": 472.8, "completions/mean_length": 484.2125, "completions/mean_terminated_length": 440.5783386230469, "completions/min_length": 398.5, "completions/min_terminated_length": 398.5, "entropy": 0.3884427772834897, "epoch": 1.1333333333333333, "frac_reward_zero_std": 0.5, "grad_norm": 1.65625, "kl": 0.04782170108519494, "learning_rate": 4.366666666666667e-06, "loss": 0.01126478835940361, "num_tokens": 858421.0, "reward": 1.1624999523162842, "reward_std": 0.31564970016479493, "rewards/JointRewardFunction/mean": 1.1624999523162842, "rewards/JointRewardFunction/std": 0.31564968526363374, "step": 170, "step_time": 23.769556162084335 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.55, "completions/max_length": 512.0, "completions/max_terminated_length": 432.3, "completions/mean_length": 475.5625, "completions/mean_terminated_length": 390.1850067138672, "completions/min_length": 405.1, "completions/min_terminated_length": 353.9, "entropy": 0.4187339372932911, "epoch": 1.2, "frac_reward_zero_std": 0.55, "grad_norm": 2.796875, "kl": 0.04600694442633539, "learning_rate": 4.033333333333333e-06, "loss": 0.004017980396747589, "num_tokens": 909270.0, "reward": 1.1249999523162841, "reward_std": 0.3038551777601242, "rewards/JointRewardFunction/mean": 1.1249999523162841, "rewards/JointRewardFunction/std": 0.303855174779892, "step": 180, "step_time": 23.616701592001483 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.775, "completions/max_length": 512.0, "completions/max_terminated_length": 327.4, "completions/mean_length": 492.325, "completions/mean_terminated_length": 300.3333343505859, "completions/min_length": 427.7, "completions/min_terminated_length": 274.1, "entropy": 0.41350143663585187, "epoch": 1.2666666666666666, "frac_reward_zero_std": 0.55, "grad_norm": 1.484375, "kl": 0.04740217045182362, "learning_rate": 3.7e-06, "loss": -0.003960463404655457, "num_tokens": 962576.0, "reward": 1.012499952316284, "reward_std": 0.3593766808509827, "rewards/JointRewardFunction/mean": 1.012499952316284, "rewards/JointRewardFunction/std": 0.35937667191028594, "step": 190, "step_time": 23.566887495698758 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.5375, "completions/max_length": 512.0, "completions/max_terminated_length": 481.9, "completions/mean_length": 478.475, "completions/mean_terminated_length": 447.27024230957034, "completions/min_length": 410.2, "completions/min_terminated_length": 410.2, "entropy": 0.4342466939240694, "epoch": 1.3333333333333333, "frac_reward_zero_std": 0.7, "grad_norm": 0.04296875, "kl": 0.05157508149277419, "learning_rate": 3.366666666666667e-06, "loss": 0.003539453446865082, "num_tokens": 1013962.0, "reward": 1.137499952316284, "reward_std": 0.25586684942245486, "rewards/JointRewardFunction/mean": 1.137499952316284, "rewards/JointRewardFunction/std": 0.2558668404817581, "step": 200, "step_time": 23.698046440214966 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.6, "completions/max_length": 512.0, "completions/max_terminated_length": 446.5, "completions/mean_length": 484.45, "completions/mean_terminated_length": 400.0850036621094, "completions/min_length": 397.9, "completions/min_terminated_length": 346.7, "entropy": 0.4431330727413297, "epoch": 1.4, "frac_reward_zero_std": 0.75, "grad_norm": 0.0284423828125, "kl": 0.049007813690695914, "learning_rate": 3.0333333333333337e-06, "loss": 0.00042528710328042506, "num_tokens": 1065994.0, "reward": 1.2124999523162843, "reward_std": 0.17422834634780884, "rewards/JointRewardFunction/mean": 1.2124999523162843, "rewards/JointRewardFunction/std": 0.1742283433675766, "step": 210, "step_time": 23.703328472495194 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.625, "completions/max_length": 512.0, "completions/max_terminated_length": 472.3, "completions/mean_length": 484.4, "completions/mean_terminated_length": 443.0000030517578, "completions/min_length": 402.3, "completions/min_terminated_length": 402.3, "entropy": 0.4779519125819206, "epoch": 1.4666666666666668, "frac_reward_zero_std": 0.45, "grad_norm": 0.103515625, "kl": 0.04795551319839433, "learning_rate": 2.7000000000000004e-06, "loss": 0.0081934854388237, "num_tokens": 1117718.0, "reward": 1.0499999523162842, "reward_std": 0.3854936808347702, "rewards/JointRewardFunction/mean": 1.0499999523162842, "rewards/JointRewardFunction/std": 0.38549367189407346, "step": 220, "step_time": 23.64517943439423 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.55, "completions/max_length": 505.9, "completions/max_terminated_length": 430.6, "completions/mean_length": 476.3, "completions/mean_terminated_length": 399.9583343505859, "completions/min_length": 416.9, "completions/min_terminated_length": 365.7, "entropy": 0.43690641410648823, "epoch": 1.5333333333333332, "frac_reward_zero_std": 0.65, "grad_norm": 0.033447265625, "kl": 0.04073553154012188, "learning_rate": 2.3666666666666667e-06, "loss": 0.004327030852437019, "num_tokens": 1168706.0, "reward": 1.0874999552965163, "reward_std": 0.28575827181339264, "rewards/JointRewardFunction/mean": 1.0874999552965163, "rewards/JointRewardFunction/std": 0.2857582658529282, "step": 230, "step_time": 23.322862002201145 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.8, "completions/max_length": 512.0, "completions/max_terminated_length": 280.4, "completions/mean_length": 496.9875, "completions/mean_terminated_length": 264.10833435058595, "completions/min_length": 454.4, "completions/min_terminated_length": 249.6, "entropy": 0.4138149328529835, "epoch": 1.6, "frac_reward_zero_std": 0.45, "grad_norm": 1.40625, "kl": 0.04409547350369394, "learning_rate": 2.0333333333333335e-06, "loss": 0.002381373755633831, "num_tokens": 1222885.0, "reward": 1.1124999523162842, "reward_std": 0.3564689517021179, "rewards/JointRewardFunction/mean": 1.1124999523162842, "rewards/JointRewardFunction/std": 0.3564689338207245, "step": 240, "step_time": 23.68468416100368 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.65, "completions/max_length": 512.0, "completions/max_terminated_length": 432.1, "completions/mean_length": 488.575, "completions/mean_terminated_length": 401.2100006103516, "completions/min_length": 418.2, "completions/min_terminated_length": 367.0, "entropy": 0.4538187999278307, "epoch": 1.6666666666666665, "frac_reward_zero_std": 0.5, "grad_norm": 1.4921875, "kl": 0.0477120385505259, "learning_rate": 1.7000000000000002e-06, "loss": -0.002869340777397156, "num_tokens": 1274999.0, "reward": 1.1249999523162841, "reward_std": 0.3211136132478714, "rewards/JointRewardFunction/mean": 1.1249999523162841, "rewards/JointRewardFunction/std": 0.3211135983467102, "step": 250, "step_time": 24.22610174829897 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.6125, "completions/max_length": 512.0, "completions/max_terminated_length": 479.0, "completions/mean_length": 475.9875, "completions/mean_terminated_length": 430.0350036621094, "completions/min_length": 382.6, "completions/min_terminated_length": 382.6, "entropy": 0.4319076906889677, "epoch": 1.7333333333333334, "frac_reward_zero_std": 0.65, "grad_norm": 1.6171875, "kl": 0.042843742744298655, "learning_rate": 1.3666666666666668e-06, "loss": 0.00379042848944664, "num_tokens": 1328918.0, "reward": 1.0999999523162842, "reward_std": 0.3038551777601242, "rewards/JointRewardFunction/mean": 1.0999999523162842, "rewards/JointRewardFunction/std": 0.303855174779892, "step": 260, "step_time": 23.86570339490718 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.5625, "completions/max_length": 512.0, "completions/max_terminated_length": 477.8, "completions/mean_length": 475.5125, "completions/mean_terminated_length": 432.52667541503905, "completions/min_length": 390.0, "completions/min_terminated_length": 390.0, "entropy": 0.4435123071074486, "epoch": 1.8, "frac_reward_zero_std": 0.65, "grad_norm": 1.59375, "kl": 0.05289563012775034, "learning_rate": 1.0333333333333333e-06, "loss": 0.004293834418058395, "num_tokens": 1379199.0, "reward": 1.137499952316284, "reward_std": 0.3076296031475067, "rewards/JointRewardFunction/mean": 1.137499952316284, "rewards/JointRewardFunction/std": 0.30762959718704225, "step": 270, "step_time": 23.709512501204152 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.625, "completions/max_length": 512.0, "completions/max_terminated_length": 482.6, "completions/mean_length": 487.675, "completions/mean_terminated_length": 450.0916687011719, "completions/min_length": 407.4, "completions/min_terminated_length": 407.4, "entropy": 0.4224535163491964, "epoch": 1.8666666666666667, "frac_reward_zero_std": 0.6, "grad_norm": 1.9296875, "kl": 0.04625407741987146, "learning_rate": 7.000000000000001e-07, "loss": 0.0024201534688472748, "num_tokens": 1431405.0, "reward": 1.1499999523162843, "reward_std": 0.2483258366584778, "rewards/JointRewardFunction/mean": 1.1499999523162843, "rewards/JointRewardFunction/std": 0.2483258366584778, "step": 280, "step_time": 23.78357428088784 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.6875, "completions/max_length": 512.0, "completions/max_terminated_length": 383.8, "completions/mean_length": 492.5875, "completions/mean_terminated_length": 359.19762268066404, "completions/min_length": 434.1, "completions/min_terminated_length": 331.7, "entropy": 0.40225436873734, "epoch": 1.9333333333333333, "frac_reward_zero_std": 0.65, "grad_norm": 1.265625, "kl": 0.045495185896288606, "learning_rate": 3.666666666666667e-07, "loss": 0.004781855642795563, "num_tokens": 1484648.0, "reward": 1.1874999523162841, "reward_std": 0.23946727216243743, "rewards/JointRewardFunction/mean": 1.1874999523162841, "rewards/JointRewardFunction/std": 0.2394672602415085, "step": 290, "step_time": 23.741302223620004 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.5875, "completions/max_length": 512.0, "completions/max_terminated_length": 432.9, "completions/mean_length": 483.4625, "completions/mean_terminated_length": 401.8050018310547, "completions/min_length": 409.8, "completions/min_terminated_length": 358.6, "entropy": 0.4492157792672515, "epoch": 2.0, "frac_reward_zero_std": 0.55, "grad_norm": 1.953125, "kl": 0.04637631082441658, "learning_rate": 3.333333333333334e-08, "loss": 0.004703029990196228, "num_tokens": 1535681.0, "reward": 1.1249999523162841, "reward_std": 0.2668025135993958, "rewards/JointRewardFunction/mean": 1.1249999523162841, "rewards/JointRewardFunction/std": 0.26680250763893126, "step": 300, "step_time": 23.57561441520811 } ], "logging_steps": 10, "max_steps": 300, "num_input_tokens_seen": 1535681, "num_train_epochs": 2, "save_steps": 10, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": true, "should_training_stop": true }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 1, "trial_name": null, "trial_params": null }