{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 2.8125, "eval_steps": 500, "global_step": 450, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 256.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.0625, "frac_reward_zero_std": 0.075, "grad_norm": 0.12458323687314987, "kl": 1.9381546007934957e-05, "learning_rate": 9.375000000000001e-06, "loss": 3.427267074584961e-08, "num_tokens": 57908.0, "reward": -1.4099999368190765, "reward_std": 0.7305359303951263, "rewards/reward_fn/mean": -1.409999930858612, "rewards/reward_fn/std": 0.8336655974388123, "step": 10 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 256.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.125, "frac_reward_zero_std": 0.05, "grad_norm": 0.13424909114837646, "kl": 3.758955572266132e-05, "learning_rate": 1.9791666666666665e-05, "loss": 2.0861625671386717e-08, "num_tokens": 115744.0, "reward": -1.5606249690055847, "reward_std": 0.6338691949844361, "rewards/reward_fn/mean": -1.5606249451637269, "rewards/reward_fn/std": 0.6913031578063965, "step": 20 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 256.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.1875, "frac_reward_zero_std": 0.025, "grad_norm": 0.12381561845541, "kl": 0.000246869730472099, "learning_rate": 3.0208333333333334e-05, "loss": 2.4400651454925537e-07, "num_tokens": 173524.0, "reward": -1.4581249356269836, "reward_std": 0.7430005550384522, "rewards/reward_fn/mean": -1.4581249117851258, "rewards/reward_fn/std": 0.7565619319677352, "step": 30 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 256.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.25, "frac_reward_zero_std": 0.025, "grad_norm": 0.14411121606826782, "kl": 0.0016107790230307727, "learning_rate": 4.0625000000000005e-05, "loss": 1.6100704669952392e-06, "num_tokens": 231416.0, "reward": -1.3331249535083771, "reward_std": 0.6972330152988434, "rewards/reward_fn/mean": -1.3331249415874482, "rewards/reward_fn/std": 0.766948527097702, "step": 40 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 256.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.3125, "frac_reward_zero_std": 0.175, "grad_norm": 0.14639103412628174, "kl": 0.007588638024753891, "learning_rate": 4.988425925925926e-05, "loss": 7.59512186050415e-06, "num_tokens": 289368.0, "reward": -1.0006249606609345, "reward_std": 0.6684236854314805, "rewards/reward_fn/mean": -1.0006249785423278, "rewards/reward_fn/std": 0.7909055948257446, "step": 50 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 256.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.375, "frac_reward_zero_std": 0.3, "grad_norm": 0.1545860469341278, "kl": 0.024985690240282566, "learning_rate": 4.8726851851851855e-05, "loss": 2.498328685760498e-05, "num_tokens": 346964.0, "reward": -0.6074999988079071, "reward_std": 0.5110155880451203, "rewards/reward_fn/mean": -0.6075000107288361, "rewards/reward_fn/std": 0.6012047231197357, "step": 60 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 256.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.4375, "frac_reward_zero_std": 0.725, "grad_norm": 0.12491069734096527, "kl": 0.042862199200317266, "learning_rate": 4.756944444444444e-05, "loss": 4.285752947907895e-05, "num_tokens": 404676.0, "reward": -0.41250000298023226, "reward_std": 0.19732050001621246, "rewards/reward_fn/mean": -0.41250000298023226, "rewards/reward_fn/std": 0.3099328398704529, "step": 70 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 256.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.5, "frac_reward_zero_std": 0.825, "grad_norm": 0.139762744307518, "kl": 0.05795831000432372, "learning_rate": 4.6412037037037034e-05, "loss": 5.795210599899292e-05, "num_tokens": 462404.0, "reward": -0.3443750023841858, "reward_std": 0.11552571803331375, "rewards/reward_fn/mean": -0.344375005364418, "rewards/reward_fn/std": 0.1971571922302246, "step": 80 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 256.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.5625, "frac_reward_zero_std": 0.9, "grad_norm": 0.0012635246384888887, "kl": 0.0758059567771852, "learning_rate": 4.525462962962963e-05, "loss": 7.580451783724129e-05, "num_tokens": 520292.0, "reward": -0.341250005364418, "reward_std": 0.06749999970197677, "rewards/reward_fn/mean": -0.34125000834465025, "rewards/reward_fn/std": 0.14239275455474854, "step": 90 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 256.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.625, "frac_reward_zero_std": 0.925, "grad_norm": 0.28265202045440674, "kl": 0.10205610543489456, "learning_rate": 4.4097222222222226e-05, "loss": 0.00010205432772636414, "num_tokens": 578164.0, "reward": -0.31062501221895217, "reward_std": 0.05374999940395355, "rewards/reward_fn/mean": -0.31062501221895217, "rewards/reward_fn/std": 0.0848927527666092, "step": 100 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 256.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.6875, "frac_reward_zero_std": 0.925, "grad_norm": 0.15435293316841125, "kl": 0.11740942746400833, "learning_rate": 4.293981481481482e-05, "loss": 0.00011740787886083126, "num_tokens": 635928.0, "reward": -0.3262500077486038, "reward_std": 0.052499999105930326, "rewards/reward_fn/mean": -0.3262500077486038, "rewards/reward_fn/std": 0.10499999821186065, "step": 110 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 256.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.75, "frac_reward_zero_std": 0.975, "grad_norm": 0.0017832271987572312, "kl": 0.1092877883464098, "learning_rate": 4.178240740740741e-05, "loss": 0.0001092874095775187, "num_tokens": 693776.0, "reward": -0.31125001013278963, "reward_std": 0.02249999940395355, "rewards/reward_fn/mean": -0.31125001013278963, "rewards/reward_fn/std": 0.04499999582767487, "step": 120 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 256.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.8125, "frac_reward_zero_std": 0.925, "grad_norm": 0.0019980475772172213, "kl": 0.1017705887556076, "learning_rate": 4.0625000000000005e-05, "loss": 0.00010176897048950196, "num_tokens": 751704.0, "reward": -0.3262500077486038, "reward_std": 0.05249999761581421, "rewards/reward_fn/mean": -0.3262500077486038, "rewards/reward_fn/std": 0.10499999821186065, "step": 130 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 256.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.875, "frac_reward_zero_std": 0.975, "grad_norm": 0.006963597144931555, "kl": 0.10655302237719297, "learning_rate": 3.94675925925926e-05, "loss": 0.00010655303485691547, "num_tokens": 809360.0, "reward": -0.31125001013278963, "reward_std": 0.02249999940395355, "rewards/reward_fn/mean": -0.31125001013278963, "rewards/reward_fn/std": 0.04499999582767487, "step": 140 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 256.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.9375, "frac_reward_zero_std": 0.925, "grad_norm": 0.0015019988641142845, "kl": 0.10568692134693265, "learning_rate": 3.831018518518519e-05, "loss": 0.00010568510042503476, "num_tokens": 867440.0, "reward": -0.33375000655651094, "reward_std": 0.06749999821186066, "rewards/reward_fn/mean": -0.33375000655651094, "rewards/reward_fn/std": 0.1349999874830246, "step": 150 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 256.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 1.0, "frac_reward_zero_std": 0.975, "grad_norm": 0.0008070049225352705, "kl": 0.07564885132014751, "learning_rate": 3.715277777777778e-05, "loss": 7.564762490801513e-05, "num_tokens": 925440.0, "reward": -0.30750001072883604, "reward_std": 0.015000000596046448, "rewards/reward_fn/mean": -0.30750001072883604, "rewards/reward_fn/std": 0.030000001192092896, "step": 160 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 256.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 1.0625, "frac_reward_zero_std": 0.925, "grad_norm": 0.0015270161675289273, "kl": 0.06843139906413853, "learning_rate": 3.5995370370370376e-05, "loss": 6.842819275334477e-05, "num_tokens": 983304.0, "reward": -0.33375000655651094, "reward_std": 0.06749999821186066, "rewards/reward_fn/mean": -0.33375000655651094, "rewards/reward_fn/std": 0.1349999874830246, "step": 170 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 256.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 1.125, "frac_reward_zero_std": 1.0, "grad_norm": 0.0007024089572951198, "kl": 0.06553028370253741, "learning_rate": 3.483796296296297e-05, "loss": 6.553027196787298e-05, "num_tokens": 1041204.0, "reward": -0.30000001192092896, "reward_std": 0.0, "rewards/reward_fn/mean": -0.30000001192092896, "rewards/reward_fn/std": 0.0, "step": 180 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 256.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 1.1875, "frac_reward_zero_std": 1.0, "grad_norm": 0.001360859489068389, "kl": 0.058797695627436045, "learning_rate": 3.368055555555556e-05, "loss": 5.8797706151381136e-05, "num_tokens": 1099072.0, "reward": -0.30000001192092896, "reward_std": 0.0, "rewards/reward_fn/mean": -0.30000001192092896, "rewards/reward_fn/std": 0.0, "step": 190 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 256.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 1.25, "frac_reward_zero_std": 1.0, "grad_norm": 0.0012780348770320415, "kl": 0.05732685485854745, "learning_rate": 3.252314814814815e-05, "loss": 5.732685676775873e-05, "num_tokens": 1156860.0, "reward": -0.30000001192092896, "reward_std": 0.0, "rewards/reward_fn/mean": -0.30000001192092896, "rewards/reward_fn/std": 0.0, "step": 200 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 256.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 1.3125, "frac_reward_zero_std": 0.95, "grad_norm": 0.0007964351680129766, "kl": 0.0754379843827337, "learning_rate": 3.136574074074074e-05, "loss": 7.543485262431204e-05, "num_tokens": 1214784.0, "reward": -0.3150000095367432, "reward_std": 0.029999998956918718, "rewards/reward_fn/mean": -0.3150000095367432, "rewards/reward_fn/std": 0.059999996423721315, "step": 210 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 256.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 1.375, "frac_reward_zero_std": 0.95, "grad_norm": 0.0004977025673724711, "kl": 0.05669140527024865, "learning_rate": 3.0208333333333334e-05, "loss": 5.669001257047057e-05, "num_tokens": 1272932.0, "reward": -0.3187500089406967, "reward_std": 0.03749999850988388, "rewards/reward_fn/mean": -0.3187500089406967, "rewards/reward_fn/std": 0.07499999701976776, "step": 220 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 256.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 1.4375, "frac_reward_zero_std": 0.975, "grad_norm": 0.0008227138314396143, "kl": 0.0492351729888469, "learning_rate": 2.9050925925925926e-05, "loss": 4.923430969938636e-05, "num_tokens": 1330828.0, "reward": -0.30750001072883604, "reward_std": 0.014999999105930329, "rewards/reward_fn/mean": -0.30750001072883604, "rewards/reward_fn/std": 0.030000001192092896, "step": 230 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 256.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 1.5, "frac_reward_zero_std": 1.0, "grad_norm": 0.0008481480763293803, "kl": 0.05081965173594653, "learning_rate": 2.789351851851852e-05, "loss": 5.0819659372791647e-05, "num_tokens": 1388588.0, "reward": -0.30000001192092896, "reward_std": 0.0, "rewards/reward_fn/mean": -0.30000001192092896, "rewards/reward_fn/std": 0.0, "step": 240 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 256.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 1.5625, "frac_reward_zero_std": 1.0, "grad_norm": 0.00036746280966326594, "kl": 0.046290718484669924, "learning_rate": 2.6736111111111112e-05, "loss": 4.6290719183161855e-05, "num_tokens": 1446308.0, "reward": -0.30000001192092896, "reward_std": 0.0, "rewards/reward_fn/mean": -0.30000001192092896, "rewards/reward_fn/std": 0.0, "step": 250 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 256.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 1.625, "frac_reward_zero_std": 1.0, "grad_norm": 0.0007307095802389085, "kl": 0.04613759703934193, "learning_rate": 2.5578703703703705e-05, "loss": 4.613760684151203e-05, "num_tokens": 1503988.0, "reward": -0.30000001192092896, "reward_std": 0.0, "rewards/reward_fn/mean": -0.30000001192092896, "rewards/reward_fn/std": 0.0, "step": 260 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 256.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 1.6875, "frac_reward_zero_std": 1.0, "grad_norm": 0.0006135129951871932, "kl": 0.051912948861718176, "learning_rate": 2.4421296296296298e-05, "loss": 5.191295058466494e-05, "num_tokens": 1561968.0, "reward": -0.30000001192092896, "reward_std": 0.0, "rewards/reward_fn/mean": -0.30000001192092896, "rewards/reward_fn/std": 0.0, "step": 270 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 256.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 1.75, "frac_reward_zero_std": 1.0, "grad_norm": 0.0008430331945419312, "kl": 0.05388685767538846, "learning_rate": 2.326388888888889e-05, "loss": 5.3886871319264175e-05, "num_tokens": 1619908.0, "reward": -0.30000001192092896, "reward_std": 0.0, "rewards/reward_fn/mean": -0.30000001192092896, "rewards/reward_fn/std": 0.0, "step": 280 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 256.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 1.8125, "frac_reward_zero_std": 1.0, "grad_norm": 0.00030109184444881976, "kl": 0.04647140288725495, "learning_rate": 2.2106481481481483e-05, "loss": 4.6471410314552486e-05, "num_tokens": 1677612.0, "reward": -0.30000001192092896, "reward_std": 0.0, "rewards/reward_fn/mean": -0.30000001192092896, "rewards/reward_fn/std": 0.0, "step": 290 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 256.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 1.875, "frac_reward_zero_std": 1.0, "grad_norm": 0.00029493175679817796, "kl": 0.04179914747364819, "learning_rate": 2.0949074074074073e-05, "loss": 4.17991541326046e-05, "num_tokens": 1735372.0, "reward": -0.30000001192092896, "reward_std": 0.0, "rewards/reward_fn/mean": -0.30000001192092896, "rewards/reward_fn/std": 0.0, "step": 300 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 256.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 1.9375, "frac_reward_zero_std": 1.0, "grad_norm": 0.0018822322599589825, "kl": 0.04672949458472431, "learning_rate": 1.9791666666666665e-05, "loss": 4.672949435189366e-05, "num_tokens": 1793268.0, "reward": -0.30000001192092896, "reward_std": 0.0, "rewards/reward_fn/mean": -0.30000001192092896, "rewards/reward_fn/std": 0.0, "step": 310 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 256.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 2.0, "frac_reward_zero_std": 1.0, "grad_norm": 0.0006305224960669875, "kl": 0.044585675746202466, "learning_rate": 1.863425925925926e-05, "loss": 4.458568291738629e-05, "num_tokens": 1850880.0, "reward": -0.30000001192092896, "reward_std": 0.0, "rewards/reward_fn/mean": -0.30000001192092896, "rewards/reward_fn/std": 0.0, "step": 320 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 256.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 2.0625, "frac_reward_zero_std": 1.0, "grad_norm": 0.0003445159236434847, "kl": 0.04190994901582599, "learning_rate": 1.747685185185185e-05, "loss": 4.190995532553643e-05, "num_tokens": 1908628.0, "reward": -0.30000001192092896, "reward_std": 0.0, "rewards/reward_fn/mean": -0.30000001192092896, "rewards/reward_fn/std": 0.0, "step": 330 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 256.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 2.125, "frac_reward_zero_std": 0.975, "grad_norm": 0.00043421241571195424, "kl": 0.04299232084304094, "learning_rate": 1.6319444444444444e-05, "loss": 4.2992207454517484e-05, "num_tokens": 1966240.0, "reward": -0.30750001072883604, "reward_std": 0.014999999105930329, "rewards/reward_fn/mean": -0.30750001072883604, "rewards/reward_fn/std": 0.030000001192092896, "step": 340 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 256.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 2.1875, "frac_reward_zero_std": 1.0, "grad_norm": 0.0005787091795355082, "kl": 0.047347473446279764, "learning_rate": 1.5162037037037038e-05, "loss": 4.7347473446279766e-05, "num_tokens": 2024200.0, "reward": -0.30000001192092896, "reward_std": 0.0, "rewards/reward_fn/mean": -0.30000001192092896, "rewards/reward_fn/std": 0.0, "step": 350 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 256.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 2.25, "frac_reward_zero_std": 1.0, "grad_norm": 0.0006160089396871626, "kl": 0.038707632245495914, "learning_rate": 1.4004629629629631e-05, "loss": 3.870762884616852e-05, "num_tokens": 2081972.0, "reward": -0.30000001192092896, "reward_std": 0.0, "rewards/reward_fn/mean": -0.30000001192092896, "rewards/reward_fn/std": 0.0, "step": 360 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 256.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 2.3125, "frac_reward_zero_std": 1.0, "grad_norm": 0.0009722487884573638, "kl": 0.04313907837495208, "learning_rate": 1.2847222222222222e-05, "loss": 4.313907993491739e-05, "num_tokens": 2139648.0, "reward": -0.30000001192092896, "reward_std": 0.0, "rewards/reward_fn/mean": -0.30000001192092896, "rewards/reward_fn/std": 0.0, "step": 370 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 256.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 2.375, "frac_reward_zero_std": 1.0, "grad_norm": 0.0004891545977443457, "kl": 0.04277103706263006, "learning_rate": 1.1689814814814815e-05, "loss": 4.277102998457849e-05, "num_tokens": 2197280.0, "reward": -0.30000001192092896, "reward_std": 0.0, "rewards/reward_fn/mean": -0.30000001192092896, "rewards/reward_fn/std": 0.0, "step": 380 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 256.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 2.4375, "frac_reward_zero_std": 1.0, "grad_norm": 0.00046821596333757043, "kl": 0.04802478738129139, "learning_rate": 1.0532407407407408e-05, "loss": 4.802478651981801e-05, "num_tokens": 2255032.0, "reward": -0.30000001192092896, "reward_std": 0.0, "rewards/reward_fn/mean": -0.30000001192092896, "rewards/reward_fn/std": 0.0, "step": 390 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 256.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 2.5, "frac_reward_zero_std": 0.975, "grad_norm": 0.0004983897670172155, "kl": 0.04196312534622848, "learning_rate": 9.375000000000001e-06, "loss": 4.196249938104302e-05, "num_tokens": 2312772.0, "reward": -0.31125001013278963, "reward_std": 0.02249999940395355, "rewards/reward_fn/mean": -0.31125001013278963, "rewards/reward_fn/std": 0.04499999582767487, "step": 400 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 256.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 2.5625, "frac_reward_zero_std": 0.975, "grad_norm": 0.00025357346748933196, "kl": 0.04060801230370999, "learning_rate": 8.217592592592594e-06, "loss": 4.060695064254105e-05, "num_tokens": 2370916.0, "reward": -0.30750001072883604, "reward_std": 0.015000000596046448, "rewards/reward_fn/mean": -0.30750001072883604, "rewards/reward_fn/std": 0.030000001192092896, "step": 410 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 256.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 2.625, "frac_reward_zero_std": 1.0, "grad_norm": 0.0007254995289258659, "kl": 0.04095169799402356, "learning_rate": 7.060185185185185e-06, "loss": 4.09516942454502e-05, "num_tokens": 2429052.0, "reward": -0.30000001192092896, "reward_std": 0.0, "rewards/reward_fn/mean": -0.30000001192092896, "rewards/reward_fn/std": 0.0, "step": 420 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 256.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 2.6875, "frac_reward_zero_std": 1.0, "grad_norm": 0.0008786269463598728, "kl": 0.04227711800485849, "learning_rate": 5.902777777777778e-06, "loss": 4.227712051942945e-05, "num_tokens": 2486904.0, "reward": -0.30000001192092896, "reward_std": 0.0, "rewards/reward_fn/mean": -0.30000001192092896, "rewards/reward_fn/std": 0.0, "step": 430 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 256.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 2.75, "frac_reward_zero_std": 0.975, "grad_norm": 0.0004273704544175416, "kl": 0.04474450033158064, "learning_rate": 4.7453703703703705e-06, "loss": 4.4743606122210625e-05, "num_tokens": 2544744.0, "reward": -0.30750001072883604, "reward_std": 0.014999999105930329, "rewards/reward_fn/mean": -0.30750001072883604, "rewards/reward_fn/std": 0.030000001192092896, "step": 440 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 256.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 2.8125, "frac_reward_zero_std": 0.975, "grad_norm": 0.0004974489565938711, "kl": 0.0426725294906646, "learning_rate": 3.5879629629629633e-06, "loss": 4.2671963456086814e-05, "num_tokens": 2602668.0, "reward": -0.31125001013278963, "reward_std": 0.02249999940395355, "rewards/reward_fn/mean": -0.31125001013278963, "rewards/reward_fn/std": 0.04499999582767487, "step": 450 } ], "logging_steps": 10, "max_steps": 480, "num_input_tokens_seen": 2602668, "num_train_epochs": 3, "save_steps": 50, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": true, "should_training_stop": false }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 4, "trial_name": null, "trial_params": null }